From 8b313272dde2f84bd7bd8fbcc5aad9bc26803d1f Mon Sep 17 00:00:00 2001 From: "kaf24@scramble.cl.cam.ac.uk" Date: Fri, 31 Dec 2004 17:26:00 +0000 Subject: [PATCH] bitkeeper revision 1.1159.1.525 (41d58ba8VgPwkfEiKrUXF8b7LLwQ5Q) Andy's control-interface message switch -- first cut. --- .rootkeys | 9 + BitKeeper/etc/ignore | 2 + .../arch/xen/kernel/ctrl_if.c | 135 +-- .../drivers/xen/evtchn/evtchn.c | 16 +- tools/Makefile | 3 + tools/libxc/xc_domain.c | 2 +- tools/misc/xend | 43 + tools/python/setup.py | 1 + tools/python/xen/lowlevel/xu/xu.c | 814 +++++++++++------ tools/python/xen/xend/server/SrvDaemon.py | 6 +- tools/python/xen/xend/server/channel.py | 9 +- tools/xcs/Makefile | 44 + tools/xcs/bindings.c | 179 ++++ tools/xcs/connection.c | 157 ++++ tools/xcs/ctrl_interface.c | 269 ++++++ tools/xcs/evtchn.c | 108 +++ tools/xcs/xcs.c | 833 ++++++++++++++++++ tools/xcs/xcs.h | 155 ++++ tools/xcs/xcs_proto.h | 101 +++ tools/xcs/xcsdump.c | 182 ++++ xen/include/public/io/domain_controller.h | 18 +- 21 files changed, 2734 insertions(+), 352 deletions(-) create mode 100644 tools/xcs/Makefile create mode 100644 tools/xcs/bindings.c create mode 100644 tools/xcs/connection.c create mode 100644 tools/xcs/ctrl_interface.c create mode 100644 tools/xcs/evtchn.c create mode 100644 tools/xcs/xcs.c create mode 100644 tools/xcs/xcs.h create mode 100644 tools/xcs/xcs_proto.h create mode 100644 tools/xcs/xcsdump.c diff --git a/.rootkeys b/.rootkeys index dfedd8bf52..0b4b69a4d7 100644 --- a/.rootkeys +++ b/.rootkeys @@ -622,6 +622,15 @@ 4194e8612TrrMvC8ZlA4h2ZYCPWz4g tools/x2d2/minixend.c 4194e861x2eqNCD61RYPCUEBVdMYuw tools/x2d2/minixend.h 4194e861A4V9VbD_FYmgXpYEj5YwVg tools/x2d2/util.c +41d58ba63w1WfBmd6Cr_18nhLNv7PA tools/xcs/Makefile +41d58ba6NxgkfzD_rmsGjgd_zJ3H_w tools/xcs/bindings.c +41d58ba6I2umi60mShq4Pl0RDg7lzQ tools/xcs/connection.c +41d58ba6YyYu53bFuoIAw9hNNmneEg tools/xcs/ctrl_interface.c +41d58ba6Ru9ZbhTjgYX_oiszSIwCww tools/xcs/evtchn.c +41d58ba6x9KO1CQBT7kKOKq_pJYC3g tools/xcs/xcs.c +41d58ba6R6foSMtSFEcu-yxWFrT8VQ tools/xcs/xcs.h +41d58ba6qyr2BkTcH2WlNBYLRyl2Yw tools/xcs/xcs_proto.h +41d58ba6ijEF6fedqRO5vFu7uCirZg tools/xcs/xcsdump.c 403a3edbrr8RE34gkbR40zep98SXbg tools/xentrace/Makefile 40a107afN60pFdURgBv9KwEzgRl5mQ tools/xentrace/formats 4050c413PhhLNAYk3TEwP37i_iLw9Q tools/xentrace/xentrace.8 diff --git a/BitKeeper/etc/ignore b/BitKeeper/etc/ignore index e9d5e9cc41..00addd679e 100644 --- a/BitKeeper/etc/ignore +++ b/BitKeeper/etc/ignore @@ -75,6 +75,8 @@ tools/vnetd/vnetd tools/web-shutdown.tap tools/x2d2/minixend tools/xentrace/xentrace +tools/xcs/xcs +tools/xcs/xcsdump tools/xfrd/xfrd xen/BLOG xen/arch/x86/asm-offsets.s diff --git a/linux-2.6.10-xen-sparse/arch/xen/kernel/ctrl_if.c b/linux-2.6.10-xen-sparse/arch/xen/kernel/ctrl_if.c index 16852cb02a..3d305718a6 100644 --- a/linux-2.6.10-xen-sparse/arch/xen/kernel/ctrl_if.c +++ b/linux-2.6.10-xen-sparse/arch/xen/kernel/ctrl_if.c @@ -46,6 +46,19 @@ #define DPRINTK(_f, _a...) ((void)0) #endif +/* + * Extra ring macros to sync a consumer index up to the public producer index. + * Generally UNSAFE, but we use it for recovery and shutdown in some cases. + */ +#define RING_DROP_PENDING_REQUESTS(_p, _r) \ + do { \ + (_r)->req_cons = (_r)->sring->req_prod; \ + } while (0) +#define RING_DROP_PENDING_RESPONSES(_p, _r) \ + do { \ + (_r)->rsp_cons = (_r)->sring->rsp_prod; \ + } while (0) + /* * Only used by initial domain which must create its own control-interface * event channel. This value is picked up by the user-space domain controller @@ -59,8 +72,8 @@ static spinlock_t ctrl_if_lock; static struct irqaction ctrl_if_irq_action; -static CONTROL_RING_IDX ctrl_if_tx_resp_cons; -static CONTROL_RING_IDX ctrl_if_rx_req_cons; +static ctrl_front_ring_t ctrl_if_tx_ring; +static ctrl_back_ring_t ctrl_if_rx_ring; /* Incoming message requests. */ /* Primary message type -> message handler. */ @@ -97,8 +110,6 @@ static void __ctrl_if_rx_tasklet(unsigned long data); static DECLARE_TASKLET(ctrl_if_rx_tasklet, __ctrl_if_rx_tasklet, 0); #define get_ctrl_if() ((control_if_t *)((char *)HYPERVISOR_shared_info + 2048)) -#define TX_FULL(_c) \ - (((_c)->tx_req_prod - ctrl_if_tx_resp_cons) == CONTROL_RING_SIZE) static void ctrl_if_notify_controller(void) { @@ -113,21 +124,20 @@ static void ctrl_if_rxmsg_default_handler(ctrl_msg_t *msg, unsigned long id) static void __ctrl_if_tx_tasklet(unsigned long data) { - control_if_t *ctrl_if = get_ctrl_if(); - ctrl_msg_t *msg; - int was_full = TX_FULL(ctrl_if); - CONTROL_RING_IDX rp; + ctrl_msg_t *msg; + int was_full = RING_FULL(CTRL_RING, &ctrl_if_tx_ring); + RING_IDX i, rp; - rp = ctrl_if->tx_resp_prod; + i = ctrl_if_tx_ring.rsp_cons; + rp = ctrl_if_tx_ring.sring->rsp_prod; rmb(); /* Ensure we see all requests up to 'rp'. */ - while ( ctrl_if_tx_resp_cons != rp ) + for ( ; i != rp; i++ ) { - msg = &ctrl_if->tx_ring[MASK_CONTROL_IDX(ctrl_if_tx_resp_cons)]; - - DPRINTK("Rx-Rsp %u/%u :: %d/%d\n", - ctrl_if_tx_resp_cons, - ctrl_if->tx_resp_prod, + msg = RING_GET_RESPONSE(CTRL_RING, &ctrl_if_tx_ring, i); + + DPRINTK("Rx-Rsp %u/%u :: %d/%d\n", i-1, + ctrl_if_tx_ring.sring->rsp_prod, msg->type, msg->subtype); /* Execute the callback handler, if one was specified. */ @@ -138,16 +148,16 @@ static void __ctrl_if_tx_tasklet(unsigned long data) smp_mb(); /* Execute, /then/ free. */ ctrl_if_txmsg_id_mapping[msg->id].fn = NULL; } - - /* - * Step over the message in the ring /after/ finishing reading it. As - * soon as the index is updated then the message may get blown away. - */ - smp_mb(); - ctrl_if_tx_resp_cons++; } - if ( was_full && !TX_FULL(ctrl_if) ) + /* + * Step over messages in the ring /after/ finishing reading them. As soon + * as the index is updated then the message may get blown away. + */ + smp_mb(); + ctrl_if_tx_ring.rsp_cons = i; + + if ( was_full && !RING_FULL(CTRL_RING, &ctrl_if_tx_ring) ) { wake_up(&ctrl_if_tx_wait); run_task_queue(&ctrl_if_tx_tq); @@ -172,24 +182,27 @@ static void __ctrl_if_rxmsg_deferred(void *unused) static void __ctrl_if_rx_tasklet(unsigned long data) { - control_if_t *ctrl_if = get_ctrl_if(); ctrl_msg_t msg, *pmsg; - CONTROL_RING_IDX rp, dp; + CONTROL_RING_IDX dp; + RING_IDX rp, i; + i = ctrl_if_rx_ring.req_cons; + rp = ctrl_if_rx_ring.sring->req_prod; dp = ctrl_if_rxmsg_deferred_prod; - rp = ctrl_if->rx_req_prod; rmb(); /* Ensure we see all requests up to 'rp'. */ - - while ( ctrl_if_rx_req_cons != rp ) + + for ( ; i != rp; i++) { - pmsg = &ctrl_if->rx_ring[MASK_CONTROL_IDX(ctrl_if_rx_req_cons++)]; + pmsg = RING_GET_REQUEST(CTRL_RING, &ctrl_if_rx_ring, i); memcpy(&msg, pmsg, offsetof(ctrl_msg_t, msg)); - DPRINTK("Rx-Req %u/%u :: %d/%d\n", - ctrl_if_rx_req_cons-1, - ctrl_if->rx_req_prod, + DPRINTK("Rx-Req %u/%u :: %d/%d\n", i-1, + ctrl_if_rx_ring.sring->req_prod, msg.type, msg.subtype); + if ( msg.length > sizeof(msg.msg) ) + msg.length = sizeof(msg.msg); + if ( msg.length != 0 ) memcpy(msg.msg, pmsg->msg, msg.length); @@ -201,6 +214,8 @@ static void __ctrl_if_rx_tasklet(unsigned long data) (*ctrl_if_rxmsg_handler[msg.type])(&msg, 0); } + ctrl_if_rx_ring.req_cons = i; + if ( dp != ctrl_if_rxmsg_deferred_prod ) { wmb(); @@ -212,12 +227,10 @@ static void __ctrl_if_rx_tasklet(unsigned long data) static irqreturn_t ctrl_if_interrupt(int irq, void *dev_id, struct pt_regs *regs) { - control_if_t *ctrl_if = get_ctrl_if(); - - if ( ctrl_if_tx_resp_cons != ctrl_if->tx_resp_prod ) + if ( RING_HAS_UNCONSUMED_RESPONSES(CTRL_RING, &ctrl_if_tx_ring) ) tasklet_schedule(&ctrl_if_tx_tasklet); - if ( ctrl_if_rx_req_cons != ctrl_if->rx_req_prod ) + if ( RING_HAS_UNCONSUMED_REQUESTS(CTRL_RING, &ctrl_if_rx_ring) ) tasklet_schedule(&ctrl_if_rx_tasklet); return IRQ_HANDLED; @@ -229,13 +242,13 @@ ctrl_if_send_message_noblock( ctrl_msg_handler_t hnd, unsigned long id) { - control_if_t *ctrl_if = get_ctrl_if(); unsigned long flags; + ctrl_msg_t *dmsg; int i; spin_lock_irqsave(&ctrl_if_lock, flags); - if ( TX_FULL(ctrl_if) ) + if ( RING_FULL(CTRL_RING, &ctrl_if_tx_ring) ) { spin_unlock_irqrestore(&ctrl_if_lock, flags); return -EAGAIN; @@ -252,14 +265,15 @@ ctrl_if_send_message_noblock( } DPRINTK("Tx-Req %u/%u :: %d/%d\n", - ctrl_if->tx_req_prod, - ctrl_if_tx_resp_cons, + ctrl_if_tx_ring.req_prod_pvt, + ctrl_if_tx_ring.rsp_cons, msg->type, msg->subtype); - memcpy(&ctrl_if->tx_ring[MASK_CONTROL_IDX(ctrl_if->tx_req_prod)], - msg, sizeof(*msg)); - wmb(); /* Write the message before letting the controller peek at it. */ - ctrl_if->tx_req_prod++; + dmsg = RING_GET_REQUEST(CTRL_RING, &ctrl_if_tx_ring, + ctrl_if_tx_ring.req_prod_pvt); + memcpy(dmsg, msg, sizeof(*msg)); + ctrl_if_tx_ring.req_prod_pvt++; + RING_PUSH_REQUESTS(CTRL_RING, &ctrl_if_tx_ring); spin_unlock_irqrestore(&ctrl_if_lock, flags); @@ -358,10 +372,8 @@ int ctrl_if_enqueue_space_callback( struct tq_struct *task) { - control_if_t *ctrl_if = get_ctrl_if(); - /* Fast path. */ - if ( !TX_FULL(ctrl_if) ) + if ( !RING_FULL(CTRL_RING, &ctrl_if_tx_ring) ) return 0; (void)queue_task(task, &ctrl_if_tx_tq); @@ -372,14 +384,13 @@ ctrl_if_enqueue_space_callback( * certainly return 'not full'. */ smp_mb(); - return TX_FULL(ctrl_if); + return RING_FULL(CTRL_RING, &ctrl_if_tx_ring); } void ctrl_if_send_response( ctrl_msg_t *msg) { - control_if_t *ctrl_if = get_ctrl_if(); unsigned long flags; ctrl_msg_t *dmsg; @@ -390,15 +401,16 @@ ctrl_if_send_response( spin_lock_irqsave(&ctrl_if_lock, flags); DPRINTK("Tx-Rsp %u :: %d/%d\n", - ctrl_if->rx_resp_prod, + ctrl_if_rx_ring.rsp_prod_pvt, msg->type, msg->subtype); - dmsg = &ctrl_if->rx_ring[MASK_CONTROL_IDX(ctrl_if->rx_resp_prod)]; + dmsg = RING_GET_RESPONSE(CTRL_RING, &ctrl_if_rx_ring, + ctrl_if_rx_ring.rsp_prod_pvt); if ( dmsg != msg ) memcpy(dmsg, msg, sizeof(*msg)); - wmb(); /* Write the message before letting the controller peek at it. */ - ctrl_if->rx_resp_prod++; + ctrl_if_rx_ring.rsp_prod_pvt++; + RING_PUSH_RESPONSES(CTRL_RING, &ctrl_if_rx_ring); spin_unlock_irqrestore(&ctrl_if_lock, flags); @@ -469,8 +481,6 @@ void ctrl_if_suspend(void) void ctrl_if_resume(void) { - control_if_t *ctrl_if = get_ctrl_if(); - if ( xen_start_info.flags & SIF_INITDOMAIN ) { /* @@ -491,8 +501,8 @@ void ctrl_if_resume(void) } /* Sync up with shared indexes. */ - ctrl_if_tx_resp_cons = ctrl_if->tx_resp_prod; - ctrl_if_rx_req_cons = ctrl_if->rx_resp_prod; + RING_DROP_PENDING_RESPONSES(CTRL_RING, &ctrl_if_tx_ring); + RING_DROP_PENDING_REQUESTS(CTRL_RING, &ctrl_if_rx_ring); ctrl_if_evtchn = xen_start_info.domain_controller_evtchn; ctrl_if_irq = bind_evtchn_to_irq(ctrl_if_evtchn); @@ -505,11 +515,15 @@ void ctrl_if_resume(void) void __init ctrl_if_init(void) { - int i; + control_if_t *ctrl_if = get_ctrl_if(); + int i; for ( i = 0; i < 256; i++ ) ctrl_if_rxmsg_handler[i] = ctrl_if_rxmsg_default_handler; + FRONT_RING_ATTACH(CTRL_RING, &ctrl_if_tx_ring, &ctrl_if->tx_ring); + BACK_RING_ATTACH(CTRL_RING, &ctrl_if_rx_ring, &ctrl_if->rx_ring); + spin_lock_init(&ctrl_if_lock); ctrl_if_resume(); @@ -532,12 +546,13 @@ __initcall(ctrl_if_late_setup); int ctrl_if_transmitter_empty(void) { - return (get_ctrl_if()->tx_req_prod == ctrl_if_tx_resp_cons); + return (ctrl_if_tx_ring.sring->req_prod == ctrl_if_tx_ring.rsp_cons); + } void ctrl_if_discard_responses(void) { - ctrl_if_tx_resp_cons = get_ctrl_if()->tx_resp_prod; + RING_DROP_PENDING_RESPONSES(CTRL_RING, &ctrl_if_tx_ring); } EXPORT_SYMBOL(ctrl_if_send_message_noblock); diff --git a/linux-2.6.10-xen-sparse/drivers/xen/evtchn/evtchn.c b/linux-2.6.10-xen-sparse/drivers/xen/evtchn/evtchn.c index 165120b401..f5da4283d1 100644 --- a/linux-2.6.10-xen-sparse/drivers/xen/evtchn/evtchn.c +++ b/linux-2.6.10-xen-sparse/drivers/xen/evtchn/evtchn.c @@ -61,8 +61,8 @@ static devfs_handle_t xen_dev_dir; struct per_user_data { /* Notification ring, accessed via /dev/xen/evtchn. */ -# define RING_SIZE 2048 /* 2048 16-bit entries */ -# define RING_MASK(_i) ((_i)&(RING_SIZE-1)) +# define EVTCHN_RING_SIZE 2048 /* 2048 16-bit entries */ +# define EVTCHN_RING_MASK(_i) ((_i)&(EVTCHN_RING_SIZE-1)) u16 *ring; unsigned int ring_cons, ring_prod, ring_overflow; @@ -86,9 +86,9 @@ void evtchn_device_upcall(int port) if ( (u = port_user[port]) != NULL ) { - if ( (u->ring_prod - u->ring_cons) < RING_SIZE ) + if ( (u->ring_prod - u->ring_cons) < EVTCHN_RING_SIZE ) { - u->ring[RING_MASK(u->ring_prod)] = (u16)port; + u->ring[EVTCHN_RING_MASK(u->ring_prod)] = (u16)port; if ( u->ring_cons == u->ring_prod++ ) { wake_up_interruptible(&u->evtchn_wait); @@ -154,10 +154,10 @@ static ssize_t evtchn_read(struct file *file, char *buf, } /* Byte lengths of two chunks. Chunk split (if any) is at ring wrap. */ - if ( ((c ^ p) & RING_SIZE) != 0 ) + if ( ((c ^ p) & EVTCHN_RING_SIZE) != 0 ) { - bytes1 = (RING_SIZE - RING_MASK(c)) * sizeof(u16); - bytes2 = RING_MASK(p) * sizeof(u16); + bytes1 = (EVTCHN_RING_SIZE - EVTCHN_RING_MASK(c)) * sizeof(u16); + bytes2 = EVTCHN_RING_MASK(p) * sizeof(u16); } else { @@ -176,7 +176,7 @@ static ssize_t evtchn_read(struct file *file, char *buf, bytes2 = count - bytes1; } - if ( copy_to_user(buf, &u->ring[RING_MASK(c)], bytes1) || + if ( copy_to_user(buf, &u->ring[EVTCHN_RING_MASK(c)], bytes1) || ((bytes2 != 0) && copy_to_user(&buf[bytes1], &u->ring[0], bytes2)) ) { rc = -EFAULT; diff --git a/tools/Makefile b/tools/Makefile index 6c9855bed7..fc79cd1d9a 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -8,6 +8,7 @@ all: $(MAKE) -C xentrace $(MAKE) -C python $(MAKE) -C xfrd + $(MAKE) -C xcs install: ifneq ($(dist),yes) @@ -21,6 +22,7 @@ endif $(MAKE) -C python install $(MAKE) -C xfrd install $(MAKE) -C sv install + $(MAKE) -C xcs install dist: $(TARGET) $(MAKE) prefix=`pwd`/../../install dist=yes install @@ -34,4 +36,5 @@ clean: $(MAKE) -C xentrace clean $(MAKE) -C python clean $(MAKE) -C xfrd clean + $(MAKE) -C xcs clean diff --git a/tools/libxc/xc_domain.c b/tools/libxc/xc_domain.c index b29e4fc823..1d34b3e405 100644 --- a/tools/libxc/xc_domain.c +++ b/tools/libxc/xc_domain.c @@ -69,7 +69,7 @@ int xc_domain_pincpu(int xc_handle, dom0_op_t op; op.cmd = DOM0_PINCPUDOMAIN; op.u.pincpudomain.domain = (domid_t)domid; - op.u.pincpudomain.exec_domain = 0; + op.u.pincpudomain.exec_domain = 0; op.u.pincpudomain.cpu = cpu; return do_dom0_op(xc_handle, &op); } diff --git a/tools/misc/xend b/tools/misc/xend index 728cb2aaab..e6235349b5 100644 --- a/tools/misc/xend +++ b/tools/misc/xend @@ -21,6 +21,12 @@ """ import os import sys +import socket +import time + +XCS_PORT = 1633 +XCS_EXEC = "/usr/sbin/xcs" +XCS_LOGFILE = "/var/log/xcs.log" # Default install path for Xen binary packages. sys.path.append('/lib/python') @@ -89,6 +95,18 @@ def check_user(): msg("Xend must be run as root.") hline() raise CheckError("invalid user") + +def xcs_running(): + """ See if the control switch is running. + """ + ret = 1 + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.connect( ("127.0.0.1", XCS_PORT) ) + except: + ret = 0 + s.close() + return (ret) def main(): try: @@ -97,6 +115,31 @@ def main(): check_user() except CheckError: sys.exit(1) + + if (not xcs_running()): + if os.fork(): + time.sleep(1) # let xcs start + else: + try: + logfile = os.open(XCS_LOGFILE, + os.O_WRONLY|os.O_APPEND|os.O_CREAT) + os.close(1) + os.dup(logfile) + os.close(2) + os.dup(logfile) + os.close(logfile) + os.execlp(XCS_EXEC, XCS_EXEC) + except: + hline() + msg("Tried to start xcs, but failed. Is it installed?") + hline() + raise CheckError("couldn't start xcs") + if (not xcs_running()): + hline() + msg("Failed to start the control interface switch.") + hline() + raise CheckError("xcs not running") + daemon = SrvDaemon.instance() if not sys.argv[1:]: print 'usage: %s {start|stop|restart}' % sys.argv[0] diff --git a/tools/python/setup.py b/tools/python/setup.py index 99069d0be4..81536989ee 100644 --- a/tools/python/setup.py +++ b/tools/python/setup.py @@ -10,6 +10,7 @@ extra_compile_args = [ "-fno-strict-aliasing", "-Wall", "-Werror" ] include_dirs = [ XEN_ROOT + "/tools/python/xen/lowlevel/xu", XEN_ROOT + "/tools/libxc", XEN_ROOT + "/tools/libxutil", + XEN_ROOT + "/tools/xcs", ] library_dirs = [ XEN_ROOT + "/tools/libxc", diff --git a/tools/python/xen/lowlevel/xu/xu.c b/tools/python/xen/lowlevel/xu/xu.c index 9a67693683..bd263886d0 100644 --- a/tools/python/xen/lowlevel/xu/xu.c +++ b/tools/python/xen/lowlevel/xu/xu.c @@ -59,6 +59,7 @@ /* Set the close-on-exec flag on a file descriptor. Doesn't currently bother * to check for errors. */ +/* static void set_cloexec(int fd) { int flags = fcntl(fd, F_GETFD, 0); @@ -69,7 +70,222 @@ static void set_cloexec(int fd) flags |= FD_CLOEXEC; fcntl(fd, F_SETFD, flags); } +*/ +/* + * *********************** XCS INTERFACE *********************** + */ + +#include +#include + +static int xcs_ctrl_fd = -1; /* control connection to the xcs server. */ +static int xcs_data_fd = -1; /* data connection to the xcs server. */ +static u32 xcs_session_id = 0; + +int xcs_ctrl_send(xcs_msg_t *msg); +int xcs_ctrl_read(xcs_msg_t *msg); +int xcs_data_send(xcs_msg_t *msg); +int xcs_data_read(xcs_msg_t *msg); + +int xcs_connect(char *ip, short port) +{ + struct sockaddr_in addr; + int ret, flags; + xcs_msg_t msg; + + if (xcs_data_fd != -1) /* already connected */ + return 0; + + xcs_ctrl_fd = socket(AF_INET, SOCK_STREAM, 0); + if (xcs_ctrl_fd < 0) + { + printf("error creating xcs socket!\n"); + goto fail; + } + + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = inet_addr(ip); + memset(&(addr.sin_zero), '\0', 8); + + ret = connect(xcs_ctrl_fd, (struct sockaddr *)&addr, + sizeof(struct sockaddr)); + if (ret < 0) + { + printf("error connecting to xcs(ctrl)! (%d)\n", errno); + goto ctrl_fd_fail; + } + + //set_cloexec(xcs_ctrl_fd); + + msg.type = XCS_CONNECT_CTRL; + msg.u.connect.session_id = xcs_session_id; + xcs_ctrl_send(&msg); + xcs_ctrl_read(&msg); /* TODO: timeout + error! */ + + if (msg.result != XCS_RSLT_OK) + { + printf("error connecting xcs control channel!\n"); + goto ctrl_fd_fail; + } + xcs_session_id = msg.u.connect.session_id; + + /* now the data connection. */ + xcs_data_fd = socket(AF_INET, SOCK_STREAM, 0); + if (xcs_data_fd < 0) + { + printf("error creating xcs data socket!\n"); + goto ctrl_fd_fail; + } + + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = inet_addr(ip); + memset(&(addr.sin_zero), '\0', 8); + + ret = connect(xcs_data_fd, (struct sockaddr *)&addr, + sizeof(struct sockaddr)); + if (ret < 0) + { + printf("error connecting to xcs(data)! (%d)\n", errno); + goto data_fd_fail; + } + + //set_cloexec(xcs_data_fd); + msg.type = XCS_CONNECT_DATA; + msg.u.connect.session_id = xcs_session_id; + xcs_data_send(&msg); + xcs_data_read(&msg); /* TODO: timeout + error! */ + + if (msg.result != XCS_RSLT_OK) + { + printf("error connecting xcs control channel!\n"); + goto ctrl_fd_fail; + } + + if ( ((flags = fcntl(xcs_data_fd, F_GETFL, 0)) < 0) || + (fcntl(xcs_data_fd, F_SETFL, flags | O_NONBLOCK) < 0) ) + { + printf("Unable to set non-blocking status on data socket."); + goto data_fd_fail; + } + + /* Haven't put type binding hooks into Xend yet. */ + /* for now, register for everything: */ + /* + msg.type = XCS_MSG_BIND; + msg.u.bind.port = PORT_WILDCARD; + msg.u.bind.type = TYPE_WILDCARD; + xcs_ctrl_send(&msg); + xcs_ctrl_read(&msg); + + if (msg.result != XCS_RSLT_OK) + { + printf("error binding!\n"); + goto data_fd_fail; + } + printf("successfully connected to xcs.\n"); + */ + return 0; + +data_fd_fail: + close(xcs_data_fd); + xcs_data_fd = -1; + +ctrl_fd_fail: + close(xcs_ctrl_fd); + xcs_ctrl_fd = -1; + +fail: + return -1; + +} + +void xcs_disconnect(void) +{ + printf("xcs_disconnect called!\n"); + close(xcs_data_fd); + xcs_data_fd = -1; + close(xcs_ctrl_fd); + xcs_ctrl_fd = -1; +} + +int xcs_ctrl_read(xcs_msg_t *msg) +{ + int ret; + + ret = read(xcs_ctrl_fd, msg, sizeof(xcs_msg_t)); + if (ret != sizeof(xcs_msg_t)) { + printf("xu-xcs: ctrl read error (%d)\n", errno); + /* TODO: set xcs_fd to -1 if the connection has been dropped. */ + } else { + printf("xu-xcs: read! fd: %d, type: %u\n", xcs_ctrl_fd, msg->type); + } + return ret; +} + +int xcs_ctrl_send(xcs_msg_t *msg) +{ + int ret; + + ret = send(xcs_ctrl_fd, msg, sizeof(xcs_msg_t), 0); + if (ret != sizeof(xcs_msg_t) ) + { + printf("xu-xcs: ctrl send error(%d)\n", errno); + /* TODO: set xcs_fd to -1 if the connection has been dropped. */ + } else { + printf("xu-xcs: sent! fd: %d, type: %u\n", xcs_ctrl_fd, msg->type); + } + return ret; +} + +int xcs_data_read(xcs_msg_t *msg) +{ + int ret; + + ret = read(xcs_data_fd, msg, sizeof(xcs_msg_t)); + if (ret != sizeof(xcs_msg_t)) { + printf("xu-xcs: ctrl read error (%d)\n", errno); + /* TODO: set xcs_fd to -1 if the connection has been dropped. */ + } + return ret; +} + +int xcs_data_send(xcs_msg_t *msg) +{ + int ret; + + ret = send(xcs_data_fd, msg, sizeof(xcs_msg_t), 0); + if (ret != sizeof(xcs_msg_t) ) + { + printf("xu-xcs: ctrl send error(%d)\n", errno); + /* TODO: set xcs_fd to -1 if the connection has been dropped. */ + } + return ret; +} + +typedef struct kme_st { + xcs_msg_t msg; + struct kme_st *next; +} xcs_msg_ent_t; + + +#define XCS_RING_SIZE 64 +static xcs_msg_ent_t *req_ring[64]; +static unsigned req_prod = 0; +static unsigned req_cons = 0; + +static xcs_msg_ent_t *rsp_ring[64]; +static unsigned rsp_prod = 0; +static unsigned rsp_cons = 0; + +#define REQ_RING_ENT(_idx) (req_ring[(_idx) % XCS_RING_SIZE]) +#define RSP_RING_ENT(_idx) (rsp_ring[(_idx) % XCS_RING_SIZE]) +#define REQ_RING_FULL ( req_prod - req_cons == XCS_RING_SIZE ) +#define RSP_RING_FULL ( rsp_prod - rsp_cons == XCS_RING_SIZE ) +#define REQ_RING_EMPTY ( req_prod == req_cons ) +#define RSP_RING_EMPTY ( rsp_prod == rsp_cons ) /* * *********************** NOTIFIER *********************** */ @@ -81,81 +297,142 @@ typedef struct { static PyObject *xu_notifier_read(PyObject *self, PyObject *args) { - xu_notifier_object *xun = (xu_notifier_object *)self; - u16 v; - int bytes; + xcs_msg_ent_t *ent; + int ret; if ( !PyArg_ParseTuple(args, "") ) return NULL; - - while ( (bytes = read(xun->evtchn_fd, &v, sizeof(v))) == -1 ) + printf("xu_notifier_read()\n"); + + while ((!REQ_RING_FULL) && (!RSP_RING_FULL)) { - if ( errno == EINTR ) + ent = (xcs_msg_ent_t *)malloc(sizeof(xcs_msg_ent_t)); + ret = xcs_data_read(&ent->msg); + + if (ret == -1) + { + free(ent); + if ( errno == EINTR ) + continue; + if ( errno == EAGAIN ) + break; + return PyErr_SetFromErrno(PyExc_IOError); + } + printf("notifier got msg type %u\n", ent->msg.type); + switch (ent->msg.type) + { + case XCS_REQUEST: + REQ_RING_ENT(req_prod) = ent; + req_prod++; + continue; + + case XCS_RESPONSE: + RSP_RING_ENT(rsp_prod) = ent; + rsp_prod++; continue; - if ( errno == EAGAIN ) - goto none; - return PyErr_SetFromErrno(PyExc_IOError); + + case XCS_VIRQ: + ret = ent->msg.u.control.local_port; + free(ent); + return PyInt_FromLong(ret); + + default: + printf("Throwing away xcs msg type: %u\n", ent->msg.type); + free(ent); + } } - if ( bytes == sizeof(v) ) - return PyInt_FromLong(v); - - none: + if (!REQ_RING_EMPTY) + { + printf("nfy: req: %d\n", + REQ_RING_ENT(req_cons)->msg.u.control.local_port); + return PyInt_FromLong(REQ_RING_ENT(req_cons)->msg.u.control.local_port); + } + + if (!RSP_RING_EMPTY) + { + printf("nfy: rsp: %d\n", + RSP_RING_ENT(rsp_cons)->msg.u.control.local_port); + return PyInt_FromLong(RSP_RING_ENT(rsp_cons)->msg.u.control.local_port); + } + + printf("nfy: returning None\n"); Py_INCREF(Py_None); return Py_None; } +/* this is now a NOOP */ static PyObject *xu_notifier_unmask(PyObject *self, PyObject *args) { - xu_notifier_object *xun = (xu_notifier_object *)self; - u16 v; - int idx; - - if ( !PyArg_ParseTuple(args, "i", &idx) ) - return NULL; - - v = (u16)idx; - - (void)write(xun->evtchn_fd, &v, sizeof(v)); - Py_INCREF(Py_None); return Py_None; } +/* this is now a NOOP */ static PyObject *xu_notifier_bind(PyObject *self, PyObject *args) { - xu_notifier_object *xun = (xu_notifier_object *)self; - int idx; - - if ( !PyArg_ParseTuple(args, "i", &idx) ) - return NULL; - - if ( ioctl(xun->evtchn_fd, EVTCHN_BIND, idx) != 0 ) - return PyErr_SetFromErrno(PyExc_IOError); - Py_INCREF(Py_None); return Py_None; } -static PyObject *xu_notifier_unbind(PyObject *self, PyObject *args) +static PyObject *xu_notifier_bind_virq(PyObject *self, + PyObject *args, PyObject *kwds) { - xu_notifier_object *xun = (xu_notifier_object *)self; - int idx; + int virq; + xcs_msg_t kmsg; - if ( !PyArg_ParseTuple(args, "i", &idx) ) + static char *kwd_list[] = { "virq", NULL }; + if ( !PyArg_ParseTupleAndKeywords(args, kwds, "i", kwd_list, &virq) ) return NULL; + + kmsg.type = XCS_VIRQ_BIND; + kmsg.u.virq.virq = virq; + xcs_ctrl_send(&kmsg); + xcs_ctrl_read(&kmsg); + + if ( kmsg.result != XCS_RSLT_OK ) + { + Py_INCREF(Py_None); + return Py_None; + } + + return PyInt_FromLong(kmsg.u.virq.port); +} - if ( ioctl(xun->evtchn_fd, EVTCHN_UNBIND, idx) != 0 ) - return PyErr_SetFromErrno(PyExc_IOError); +static PyObject *xu_notifier_virq_send(PyObject *self, + PyObject *args, PyObject *kwds) +{ + int port; + xcs_msg_t kmsg; + static char *kwd_list[] = { "port", NULL }; + if ( !PyArg_ParseTupleAndKeywords(args, kwds, "i", kwd_list, &port) ) + return NULL; + + kmsg.type = XCS_VIRQ; + kmsg.u.control.local_port = port; + xcs_ctrl_send(&kmsg); + xcs_ctrl_read(&kmsg); + + if ( kmsg.result != XCS_RSLT_OK ) + { + Py_INCREF(Py_None); + return Py_None; + } + + return PyInt_FromLong(kmsg.u.virq.port); +} + +/* this is now a NOOP */ +static PyObject *xu_notifier_unbind(PyObject *self, PyObject *args) +{ Py_INCREF(Py_None); return Py_None; } static PyObject *xu_notifier_fileno(PyObject *self, PyObject *args) { - xu_notifier_object *xun = (xu_notifier_object *)self; - return PyInt_FromLong(xun->evtchn_fd); + return PyInt_FromLong(xcs_data_fd); } static PyMethodDef xu_notifier_methods[] = { @@ -178,6 +455,18 @@ static PyMethodDef xu_notifier_methods[] = { (PyCFunction)xu_notifier_unbind, METH_VARARGS, "No longer get notifications for a @port.\n" }, + + { "bind_virq", + (PyCFunction)xu_notifier_bind_virq, + METH_VARARGS | METH_KEYWORDS, + "Get notifications for a virq.\n" + " virq [int]: VIRQ to bind.\n\n" }, + + { "virq_send", + (PyCFunction)xu_notifier_virq_send, + METH_VARARGS | METH_KEYWORDS, + "Fire a virq notification.\n" + " port [int]: port that VIRQ is bound to.\n\n" }, { "fileno", (PyCFunction)xu_notifier_fileno, @@ -189,35 +478,23 @@ static PyMethodDef xu_notifier_methods[] = { staticforward PyTypeObject xu_notifier_type; +/* connect to xcs if we aren't already, and return a dummy object. */ static PyObject *xu_notifier_new(PyObject *self, PyObject *args) { xu_notifier_object *xun; - struct stat st; + int i; +printf("xu_notifier_new()\n"); if ( !PyArg_ParseTuple(args, "") ) return NULL; xun = PyObject_New(xu_notifier_object, &xu_notifier_type); - /* Make sure any existing device file links to correct device. */ - if ( (lstat(EVTCHN_DEV_NAME, &st) != 0) || - !S_ISCHR(st.st_mode) || - (st.st_rdev != makedev(EVTCHN_DEV_MAJOR, EVTCHN_DEV_MINOR)) ) - (void)unlink(EVTCHN_DEV_NAME); - - reopen: - xun->evtchn_fd = open(EVTCHN_DEV_NAME, O_NONBLOCK|O_RDWR); - if ( xun->evtchn_fd == -1 ) - { - if ( (errno == ENOENT) && - ((mkdir("/dev/xen", 0755) == 0) || (errno == EEXIST)) && - (mknod(EVTCHN_DEV_NAME, S_IFCHR|0600, - makedev(EVTCHN_DEV_MAJOR,EVTCHN_DEV_MINOR)) == 0) ) - goto reopen; - PyObject_Del((PyObject *)xun); - return PyErr_SetFromErrno(PyExc_IOError); - } - set_cloexec(xun->evtchn_fd); + for (i = 0; i < XCS_RING_SIZE; i++) + REQ_RING_ENT(i) = RSP_RING_ENT(i) = NULL; + + (void)xcs_connect("127.0.0.1", XCS_TCP_PORT); + return (PyObject *)xun; } @@ -229,8 +506,6 @@ static PyObject *xu_notifier_getattr(PyObject *obj, char *name) static void xu_notifier_dealloc(PyObject *self) { - xu_notifier_object *xun = (xu_notifier_object *)self; - (void)close(xun->evtchn_fd); PyObject_Del(self); } @@ -696,43 +971,20 @@ static PyTypeObject xu_message_type = { * *********************** PORT *********************** */ -static control_if_t *map_control_interface(int fd, unsigned long pfn, - u32 dom) -{ - char *vaddr = xc_map_foreign_range( fd, dom, PAGE_SIZE, - PROT_READ|PROT_WRITE, pfn ); - if ( vaddr == NULL ) - return NULL; - return (control_if_t *)(vaddr + 2048); -} -static void unmap_control_interface(int fd, control_if_t *c) -{ - char *vaddr = (char *)c - 2048; - (void)munmap(vaddr, PAGE_SIZE); -} - typedef struct xu_port_object { PyObject_HEAD; int xc_handle; int connected; u32 remote_dom; int local_port, remote_port; - control_if_t *interface; - CONTROL_RING_IDX tx_req_cons, tx_resp_prod; - CONTROL_RING_IDX rx_req_prod, rx_resp_cons; + struct xu_port_object *fix_next; } xu_port_object; static PyObject *port_error; +/* now a NOOP */ static PyObject *xu_port_notify(PyObject *self, PyObject *args) { - xu_port_object *xup = (xu_port_object *)self; - - if ( !PyArg_ParseTuple(args, "") ) - return NULL; - - (void)xc_evtchn_send(xup->xc_handle, xup->local_port); - Py_INCREF(Py_None); return Py_None; } @@ -741,39 +993,51 @@ static PyObject *xu_port_read_request(PyObject *self, PyObject *args) { xu_port_object *xup = (xu_port_object *)self; xu_message_object *xum; - CONTROL_RING_IDX c = xup->tx_req_cons; - control_if_t *cif = xup->interface; control_msg_t *cmsg; - - if ( !PyArg_ParseTuple(args, "") ) - return NULL; - - if ( (c == cif->tx_req_prod) || - ((c - xup->tx_resp_prod) == CONTROL_RING_SIZE) ) - { - PyErr_SetString(port_error, "no request to read"); - return NULL; + unsigned i; + xcs_msg_ent_t *ent = NULL; + + for ( i = req_cons; (i != req_prod); i++ ) { + ent = REQ_RING_ENT(i); + if (ent == NULL) + continue; + if (ent->msg.u.control.remote_dom == xup->remote_dom) + break; } + + if ((ent == NULL) || + (ent->msg.u.control.remote_dom != xup->remote_dom)) + goto none; + +printf("read request (%d:%d)\n", ent->msg.u.control.msg.type, + ent->msg.u.control.msg.subtype); - /* Need to ensure we see the request, despite seeing the index update.*/ - rmb(); - - cmsg = &cif->tx_ring[MASK_CONTROL_IDX(c)]; + cmsg = &ent->msg.u.control.msg; xum = PyObject_New(xu_message_object, &xu_message_type); memcpy(&xum->msg, cmsg, sizeof(*cmsg)); if ( xum->msg.length > sizeof(xum->msg.msg) ) xum->msg.length = sizeof(xum->msg.msg); - xup->tx_req_cons++; + free(ent); + + /* remove the entry from the ring and advance the consumer if possible */ + REQ_RING_ENT(i) = NULL; + while ( (REQ_RING_ENT(req_cons) == NULL) && (!REQ_RING_EMPTY) ) + req_cons++; + return (PyObject *)xum; + +none: +printf("read request - NO REQUEST!\n"); + Py_INCREF(Py_None); + return Py_None; + } static PyObject *xu_port_write_request(PyObject *self, PyObject *args) { xu_port_object *xup = (xu_port_object *)self; xu_message_object *xum; - CONTROL_RING_IDX p = xup->rx_req_prod; - control_if_t *cif = xup->interface; - control_msg_t *cmsg; + xcs_msg_t kmsg; if ( !PyArg_ParseTuple(args, "O", (PyObject **)&xum) ) return NULL; @@ -784,18 +1048,11 @@ static PyObject *xu_port_write_request(PyObject *self, PyObject *args) return NULL; } - if ( ((p - xup->rx_resp_cons) == CONTROL_RING_SIZE) ) - { - PyErr_SetString(port_error, "no space to write request"); - return NULL; - } - - cmsg = &cif->rx_ring[MASK_CONTROL_IDX(p)]; - memcpy(cmsg, &xum->msg, sizeof(*cmsg)); - - wmb(); - xup->rx_req_prod = cif->rx_req_prod = p + 1; - + kmsg.type = XCS_REQUEST; + kmsg.u.control.remote_dom = xup->remote_dom; + memcpy(&kmsg.u.control.msg, &xum->msg, sizeof(control_msg_t)); + xcs_data_send(&kmsg); + Py_INCREF(Py_None); return Py_None; } @@ -804,38 +1061,51 @@ static PyObject *xu_port_read_response(PyObject *self, PyObject *args) { xu_port_object *xup = (xu_port_object *)self; xu_message_object *xum; - CONTROL_RING_IDX c = xup->rx_resp_cons; - control_if_t *cif = xup->interface; control_msg_t *cmsg; - - if ( !PyArg_ParseTuple(args, "") ) - return NULL; - - if ( (c == cif->rx_resp_prod) || (c == xup->rx_req_prod) ) - { - PyErr_SetString(port_error, "no response to read"); - return NULL; + unsigned i; + xcs_msg_ent_t *ent = NULL; + + for ( i = rsp_cons; (i != rsp_prod); i++ ) { + ent = RSP_RING_ENT(i); + if (ent == NULL) + continue; + if (ent->msg.u.control.remote_dom == xup->remote_dom) + break; } + + if ((ent == NULL) || + (ent->msg.u.control.remote_dom != xup->remote_dom)) + goto none; + +printf("read response (%d:%d)\n", ent->msg.u.control.msg.type, + ent->msg.u.control.msg.subtype); - /* Need to ensure we see the response, despite seeing the index update.*/ - rmb(); - - cmsg = &cif->rx_ring[MASK_CONTROL_IDX(c)]; + cmsg = &ent->msg.u.control.msg; xum = PyObject_New(xu_message_object, &xu_message_type); memcpy(&xum->msg, cmsg, sizeof(*cmsg)); if ( xum->msg.length > sizeof(xum->msg.msg) ) xum->msg.length = sizeof(xum->msg.msg); - xup->rx_resp_cons++; + free(ent); + + /* remove the entry from the ring and advance the consumer if possible */ + RSP_RING_ENT(i) = NULL; + while ( (RSP_RING_ENT(rsp_cons) == NULL) && (!RSP_RING_EMPTY) ) + rsp_cons++; + return (PyObject *)xum; + +none: +printf("read response - NO RESPONSE!\n"); + Py_INCREF(Py_None); + return Py_None; + } static PyObject *xu_port_write_response(PyObject *self, PyObject *args) { xu_port_object *xup = (xu_port_object *)self; xu_message_object *xum; - CONTROL_RING_IDX p = xup->tx_resp_prod; - control_if_t *cif = xup->interface; - control_msg_t *cmsg; + xcs_msg_t kmsg; if ( !PyArg_ParseTuple(args, "O", (PyObject **)&xum) ) return NULL; @@ -846,17 +1116,10 @@ static PyObject *xu_port_write_response(PyObject *self, PyObject *args) return NULL; } - if ( p == xup->tx_req_cons ) - { - PyErr_SetString(port_error, "no space to write response"); - return NULL; - } - - cmsg = &cif->tx_ring[MASK_CONTROL_IDX(p)]; - memcpy(cmsg, &xum->msg, sizeof(*cmsg)); - - wmb(); - xup->tx_resp_prod = cif->tx_resp_prod = p + 1; + kmsg.type = XCS_RESPONSE; + kmsg.u.control.remote_dom = xup->remote_dom; + memcpy(&kmsg.u.control.msg, &xum->msg, sizeof(control_msg_t)); + xcs_data_send(&kmsg); Py_INCREF(Py_None); return Py_None; @@ -864,133 +1127,141 @@ static PyObject *xu_port_write_response(PyObject *self, PyObject *args) static PyObject *xu_port_request_to_read(PyObject *self, PyObject *args) { - xu_port_object *xup = (xu_port_object *)self; - CONTROL_RING_IDX c = xup->tx_req_cons; - control_if_t *cif = xup->interface; + xu_port_object *xup = (xu_port_object *)self; + xcs_msg_ent_t *ent; + int found = 0; + unsigned i; +printf("xu_port_request_to_read()\n"); if ( !PyArg_ParseTuple(args, "") ) return NULL; - if ( (c == cif->tx_req_prod) || - ((c - xup->tx_resp_prod) == CONTROL_RING_SIZE) ) - return PyInt_FromLong(0); - - return PyInt_FromLong(1); + for ( i = req_cons; (i != req_prod); i++ ) { + ent = REQ_RING_ENT(i); + if (ent == NULL) + continue; + if (ent->msg.u.control.remote_dom == xup->remote_dom) { + found = 1; + break; + } + } + + return PyInt_FromLong(found); } static PyObject *xu_port_space_to_write_request(PyObject *self, PyObject *args) { - xu_port_object *xup = (xu_port_object *)self; - CONTROL_RING_IDX p = xup->rx_req_prod; - if ( !PyArg_ParseTuple(args, "") ) return NULL; - if ( ((p - xup->rx_resp_cons) == CONTROL_RING_SIZE) ) - return PyInt_FromLong(0); - return PyInt_FromLong(1); } static PyObject *xu_port_response_to_read(PyObject *self, PyObject *args) { - xu_port_object *xup = (xu_port_object *)self; - CONTROL_RING_IDX c = xup->rx_resp_cons; - control_if_t *cif = xup->interface; + xu_port_object *xup = (xu_port_object *)self; + xcs_msg_ent_t *ent; + int found = 0; + unsigned i; +printf("xu_port_response_to_read()\n"); if ( !PyArg_ParseTuple(args, "") ) return NULL; - if ( (c == cif->rx_resp_prod) || (c == xup->rx_req_prod) ) - return PyInt_FromLong(0); - - return PyInt_FromLong(1); + for ( i = rsp_cons; (i != rsp_prod); i++ ) { + ent = RSP_RING_ENT(i); + if (ent == NULL) + continue; + if (ent->msg.u.control.remote_dom == xup->remote_dom) { + found = 1; + break; + } + } + + return PyInt_FromLong(found); } static PyObject *xu_port_space_to_write_response( PyObject *self, PyObject *args) { - xu_port_object *xup = (xu_port_object *)self; - CONTROL_RING_IDX p = xup->tx_resp_prod; - if ( !PyArg_ParseTuple(args, "") ) return NULL; - if ( p == xup->tx_req_cons ) - return PyInt_FromLong(0); - return PyInt_FromLong(1); } -static int __xu_port_connect(xu_port_object *xup) +/* NOOP */ +static PyObject *xu_port_connect(PyObject *self, PyObject *args) { - xc_dominfo_t info; - - if ( xup->connected ) - { - return 0; - } - - if ( (xc_domain_getinfo(xup->xc_handle, xup->remote_dom, 1, &info) != 1) || - (info.domid != xup->remote_dom) ) - { - PyErr_SetString(port_error, "Failed to obtain domain status"); - return -1; - } - - xup->interface = - map_control_interface(xup->xc_handle, info.shared_info_frame, - xup->remote_dom); - - if ( xup->interface == NULL ) - { - PyErr_SetString(port_error, "Failed to map domain control interface"); - return -1; - } - - /* Synchronise ring indexes. */ - xup->tx_resp_prod = xup->interface->tx_resp_prod; - xup->tx_req_cons = xup->interface->tx_resp_prod; - xup->rx_req_prod = xup->interface->rx_req_prod; - xup->rx_resp_cons = xup->interface->rx_resp_prod; - - xup->connected = 1; - - return 0; + Py_INCREF(Py_None); + return Py_None; } -static void __xu_port_disconnect(xu_port_object *xup) +/* NOOP */ +static PyObject *xu_port_disconnect(PyObject *self, PyObject *args) { - if ( xup->connected ) - unmap_control_interface(xup->xc_handle, xup->interface); - xup->connected = 0; + Py_INCREF(Py_None); + return Py_None; } -static PyObject *xu_port_connect(PyObject *self, PyObject *args) +static PyObject *xu_port_register(PyObject *self, PyObject *args, + PyObject *kwds) { - xu_port_object *xup = (xu_port_object *)self; - - if ( !PyArg_ParseTuple(args, "") ) - return NULL; + int type; + xcs_msg_t msg; + xu_port_object *xup = (xu_port_object *)self; + static char *kwd_list[] = { "type", NULL }; - if ( __xu_port_connect(xup) != 0 ) + if ( !PyArg_ParseTupleAndKeywords(args, kwds, "i", kwd_list, + &type) ) return NULL; - - Py_INCREF(Py_None); - return Py_None; + + printf("REGISTER : Dom: %3d Port: %3d Type:%3d\n", + xup->remote_dom, xup->local_port, type); + + msg.type = XCS_MSG_BIND; + msg.u.bind.port = xup->local_port; + msg.u.bind.type = type; + xcs_ctrl_send(&msg); + xcs_ctrl_read(&msg); + + if (msg.result != XCS_RSLT_OK) + { + printf(" : REGISTRATION FAILED! (%d)\n", msg.result); + return PyInt_FromLong(0); + } + + return PyInt_FromLong(1); } -static PyObject *xu_port_disconnect(PyObject *self, PyObject *args) +static PyObject *xu_port_deregister(PyObject *self, PyObject *args, + PyObject *kwds) { - xu_port_object *xup = (xu_port_object *)self; + int type; + xcs_msg_t msg; + xu_port_object *xup = (xu_port_object *)self; + static char *kwd_list[] = { "type", NULL }; - if ( !PyArg_ParseTuple(args, "") ) + if ( !PyArg_ParseTupleAndKeywords(args, kwds, "i", kwd_list, + &type) ) return NULL; - - __xu_port_disconnect(xup); - - Py_INCREF(Py_None); - return Py_None; + + printf("DEREGISTER: Dom: %3d Port: %3d Type:%3d\n", + xup->remote_dom, xup->local_port, type); + + msg.type = XCS_MSG_UNBIND; + msg.u.bind.port = xup->local_port; + msg.u.bind.type = type; + xcs_ctrl_send(&msg); + xcs_ctrl_read(&msg); + + if (msg.result != XCS_RSLT_OK) + { + printf(" : DEREGISTRATION FAILED! (%d)\n", msg.result); + return PyInt_FromLong(0); + } + + return PyInt_FromLong(1); } static PyMethodDef xu_port_methods[] = { @@ -1038,6 +1309,16 @@ static PyMethodDef xu_port_methods[] = { (PyCFunction)xu_port_space_to_write_response, METH_VARARGS, "Returns TRUE if there is space to write a response message.\n" }, + + { "register", + (PyCFunction)xu_port_register, + METH_VARARGS | METH_KEYWORDS, + "Register to receive a type of message on this channel.\n" }, + + { "deregister", + (PyCFunction)xu_port_deregister, + METH_VARARGS | METH_KEYWORDS, + "Stop receiving a type of message on this port.\n" }, { "connect", (PyCFunction)xu_port_connect, @@ -1059,6 +1340,7 @@ static PyObject *xu_port_new(PyObject *self, PyObject *args, PyObject *kwds) xu_port_object *xup; u32 dom; int port1 = 0, port2 = 0; + xcs_msg_t kmsg; static char *kwd_list[] = { "dom", "local_port", "remote_port", NULL }; @@ -1070,51 +1352,26 @@ static PyObject *xu_port_new(PyObject *self, PyObject *args, PyObject *kwds) xup->connected = 0; xup->remote_dom = dom; - - if ( (xup->xc_handle = xc_interface_open()) == -1 ) - { - PyErr_SetString(port_error, "Could not open Xen control interface"); + + kmsg.type = XCS_CIF_NEW_CC; + kmsg.u.interface.dom = xup->remote_dom; + kmsg.u.interface.local_port = port1; + kmsg.u.interface.remote_port = port2; + xcs_ctrl_send(&kmsg); + xcs_ctrl_read(&kmsg); + + if ( kmsg.result != XCS_RSLT_OK ) goto fail1; - } - - if ( dom == 0 ) - { - /* - * The control-interface event channel for DOM0 is already set up. - * We use an ioctl to discover the port at our end of the channel. - */ - port1 = ioctl(xup->xc_handle, IOCTL_PRIVCMD_INITDOMAIN_EVTCHN, NULL); - port2 = -1; /* We don't need the remote end of the DOM0 link. */ - if ( port1 < 0 ) - { - PyErr_SetString(port_error, "Could not open channel to DOM0"); - goto fail2; - } - } - else if ( xc_evtchn_bind_interdomain(xup->xc_handle, - DOMID_SELF, dom, - &port1, &port2) != 0 ) - { - PyErr_SetString(port_error, "Could not open channel to domain"); - goto fail2; - } - - xup->local_port = port1; - xup->remote_port = port2; - - if ( __xu_port_connect(xup) != 0 ) - goto fail3; - + + xup->local_port = kmsg.u.interface.local_port; + xup->remote_port = kmsg.u.interface.remote_port; + xup->connected = 1; + return (PyObject *)xup; - - fail3: - if ( dom != 0 ) - (void)xc_evtchn_close(xup->xc_handle, DOMID_SELF, port1); - fail2: - (void)xc_interface_close(xup->xc_handle); + fail1: PyObject_Del((PyObject *)xup); - return NULL; + return NULL; } static PyObject *xu_port_getattr(PyObject *obj, char *name) @@ -1131,11 +1388,20 @@ static PyObject *xu_port_getattr(PyObject *obj, char *name) static void xu_port_dealloc(PyObject *self) { + xu_port_object *xup = (xu_port_object *)self; - __xu_port_disconnect(xup); + xcs_msg_t kmsg; + if ( xup->remote_dom != 0 ) - (void)xc_evtchn_close(xup->xc_handle, DOMID_SELF, xup->local_port); - (void)xc_interface_close(xup->xc_handle); + { + kmsg.type = XCS_CIF_FREE_CC; + kmsg.u.interface.dom = xup->remote_dom; + kmsg.u.interface.local_port = xup->local_port; + kmsg.u.interface.remote_port = xup->remote_port; + xcs_ctrl_send(&kmsg); + xcs_ctrl_read(&kmsg); + } + PyObject_Del(self); } diff --git a/tools/python/xen/xend/server/SrvDaemon.py b/tools/python/xen/xend/server/SrvDaemon.py index 338d6e4d4a..6059e6de34 100644 --- a/tools/python/xen/xend/server/SrvDaemon.py +++ b/tools/python/xen/xend/server/SrvDaemon.py @@ -118,9 +118,9 @@ class NotifierPort(abstract.FileDescriptor): if hasattr(self, 'protocol'): self.protocol.doStop() self.connected = 0 - #self.notifier.close() # Not implemented. - os.close(self.fileno()) - del self.notifier + #self.notifier.close() # (this said:) Not implemented. + #os.close(self.fileno()) # But yes it is... + del self.notifier # ...as _dealloc! if hasattr(self, 'd'): self.d.callback(None) del self.d diff --git a/tools/python/xen/xend/server/channel.py b/tools/python/xen/xend/server/channel.py index 127f38f2c0..6dfebe37be 100755 --- a/tools/python/xen/xend/server/channel.py +++ b/tools/python/xen/xend/server/channel.py @@ -171,8 +171,10 @@ class VirqChannel(BaseChannel): """ BaseChannel.__init__(self, factory) self.virq = virq + self.factory = factory # Notification port (int). - self.port = xc.evtchn_bind_virq(virq) + #self.port = xc.evtchn_bind_virq(virq) + self.port = factory.notifier.bind_virq(virq) self.idx = self.port # Clients to call when a virq arrives. self.clients = [] @@ -208,7 +210,8 @@ class VirqChannel(BaseChannel): c.virqReceived(self.virq) def notify(self): - xc.evtchn_send(self.port) + # xc.evtchn_send(self.port) + self.factory.notifier.virq_send(self.port) class Channel(BaseChannel): @@ -279,6 +282,7 @@ class Channel(BaseChannel): self.devs.append(dev) for ty in types: self.devs_by_type[ty] = dev + self.port.register(ty) def deregisterDevice(self, dev): """Remove the registration for a device controller. @@ -290,6 +294,7 @@ class Channel(BaseChannel): types = [ ty for (ty, d) in self.devs_by_type.items() if d == dev ] for ty in types: del self.devs_by_type[ty] + self.port.deregister(ty) def getDevice(self, type): """Get the device controller handling a message type. diff --git a/tools/xcs/Makefile b/tools/xcs/Makefile new file mode 100644 index 0000000000..7fc8286991 --- /dev/null +++ b/tools/xcs/Makefile @@ -0,0 +1,44 @@ +# Makefile for XCS +# Andrew Warfield, 2004 + +XEN_ROOT=../.. +include $(XEN_ROOT)/tools/Make.defs + +XCS_INSTALL_DIR = /usr/sbin + +CC = gcc +CFLAGS = -Wall -Werror -g3 -D _XOPEN_SOURCE=600 + +CFLAGS += -I $(XEN_XC) +CFLAGS += -I $(XEN_LIBXC) +CFLAGS += -I $(XEN_LIBXUTIL) + +SRCS := +SRCS += ctrl_interface.c +SRCS += bindings.c +SRCS += connection.c +SRCS += evtchn.c +SRCS += xcs.c + +HDRS = $(wildcard *.h) +OBJS = $(patsubst %.c,%.o,$(SRCS)) +BIN = xcs + +all: $(BIN) xcsdump + +clean: + $(RM) *.a *.so *.o *.rpm $(BIN) ctrl_dump + +xcsdump: xcsdump.c + $(CC) $(CFLAGS) -o xcsdump xcsdump.c -L$(XEN_LIBXC) -L$(XEN_LIBXUTIL) \ + ctrl_interface.c evtchn.c -lxc -lxutil + +$(BIN): $(OBJS) + $(CC) $(CFLAGS) $^ -o $@ -L$(XEN_LIBXC) -L$(XEN_LIBXUTIL) -lxc -lxutil + +install: xcs xcsdump + mkdir -p $(prefix)/$(XCS_INSTALL_DIR) + mkdir -p $(prefix)/usr/include + install -m0755 xcs $(prefix)/$(XCS_INSTALL_DIR) + install -m0755 xcsdump $(prefix)/$(XCS_INSTALL_DIR) + install -m0644 xcs_proto.h $(prefix)/usr/include diff --git a/tools/xcs/bindings.c b/tools/xcs/bindings.c new file mode 100644 index 0000000000..9b09f51568 --- /dev/null +++ b/tools/xcs/bindings.c @@ -0,0 +1,179 @@ +/* bindings.c + * + * Manage subscriptions for the control interface switch. + * + * (c) 2004, Andrew Warfield + * + */ + +/* Interfaces: + * + * xcs_bind (port, type, connection) + * - Register connection to receive messages of this type. + * xcs_unbind (port, type, connection) + * - Remove an existing registration. (Must be an exact match) + * xcs_lookup (port, type) + * - Return a list of connections matching a registration. + * + * - All connections have a connection.bindings list of current bindings. + * - (port, type) pairs may be wildcarded with -1. + */ + +#include +#include +#include +#include +#include "xcs.h" + + +typedef struct binding_ent_st { + connection_t *con; + struct binding_ent_st *next; +} binding_ent_t; + +#define BINDING_TABLE_SIZE 1024 + +static binding_ent_t *binding_table[BINDING_TABLE_SIZE]; + +#define PORT_WILD(_ent) ((_ent)->port == PORT_WILDCARD) +#define TYPE_WILD(_ent) ((_ent)->type == TYPE_WILDCARD) +#define FULLY_WILD(_ent) (PORT_WILD(_ent) && TYPE_WILD(_ent)) + +#define BINDING_HASH(_key) \ + ((((_key)->port * 11) ^ (_key)->type) % BINDING_TABLE_SIZE) + + +void init_bindings(void) +{ + memset(binding_table, 0, sizeof(binding_table)); +} + +static int table_add(binding_ent_t *table[], + connection_t *con, + binding_key_t *key) +{ + binding_ent_t **curs, *ent; + + curs = &table[BINDING_HASH(key)]; + + while (*curs != NULL) { + if ((*curs)->con == con) { + DPRINTF("Tried to add an ent that already existed.\n"); + goto done; + } + curs = &(*curs)->next; + } + + if (connection_add_binding(con, key) != 0) + { + DPRINTF("couldn't add binding on connection (%lu)\n", con->id); + goto fail; + } + ent = (binding_ent_t *)malloc(sizeof(binding_ent_t)); + if (ent == 0) { + DPRINTF("couldn't alloc binding ent!\n"); + goto fail; + } + ent->con = con; + ent->next = NULL; + *curs = ent; + +done: + return 0; + +fail: + return -1; +} + + +static inline int binding_has_colliding_hashes(connection_t *con, + binding_key_t *key) +{ + int hash, count = 0; + binding_key_ent_t *ent; + + ent = con->bindings; + hash = BINDING_HASH(key); + + while (ent != NULL) { + if (BINDING_HASH(&ent->key) == hash) count ++; + ent = ent->next; + } + + return (count > 1); +} +static int table_remove(binding_ent_t *table[], + connection_t *con, + binding_key_t *key) +{ + binding_ent_t **curs, *ent; + + if (!binding_has_colliding_hashes(con, key)) + { + + curs = &table[BINDING_HASH(key)]; + + while ((*curs != NULL) && ((*curs)->con != con)) + curs = &(*curs)->next; + + if (*curs != NULL) { + ent = *curs; + *curs = (*curs)->next; + free(ent); + } + } + + connection_remove_binding(con, key); + + return 0; +} + +int xcs_bind(connection_t *con, int port, u16 type) +{ + binding_key_t key; + + key.port = port; + key.type = type; + + return table_add(binding_table, con, &key); +} + +int xcs_unbind(connection_t *con, int port, u16 type) +{ + binding_key_t key; + + key.port = port; + key.type = type; + + return table_remove(binding_table, con, &key); +} + + +static void for_each_binding(binding_ent_t *list, binding_key_t *key, + void (*f)(connection_t *, void *), void *arg) +{ + while (list != NULL) + { + if (connection_has_binding(list->con, key)) + f(list->con, arg); + list = list->next; + } +} + +void xcs_lookup(int port, u16 type, void (*f)(connection_t *, void *), + void *arg) +{ + binding_key_t key; + + key.port = port; key.type = type; + for_each_binding(binding_table[BINDING_HASH(&key)], &key, f, arg); + + key.port = port; key.type = TYPE_WILDCARD; + for_each_binding(binding_table[BINDING_HASH(&key)], &key, f, arg); + + key.port = PORT_WILDCARD; key.type = type; + for_each_binding(binding_table[BINDING_HASH(&key)], &key, f, arg); + + key.port = PORT_WILDCARD; key.type = TYPE_WILDCARD; + for_each_binding(binding_table[BINDING_HASH(&key)], &key, f, arg); +} diff --git a/tools/xcs/connection.c b/tools/xcs/connection.c new file mode 100644 index 0000000000..3b5747de68 --- /dev/null +++ b/tools/xcs/connection.c @@ -0,0 +1,157 @@ +/* + * connection.c + * + * State associated with a client connection to xcs. + * + * Copyright (c) 2004, Andrew Warfield + */ + +#include +#include +#include +#include "xcs.h" + +connection_t *connection_list = NULL; + +#define CONNECTED(_c) (((_c)->ctrl_fd != -1) || ((_c)->data_fd != -1)) + +connection_t *get_con_by_session(unsigned long session_id) +{ + connection_t **c, *ent = NULL; + + c = &connection_list; + + DPRINTF("looking for id: %lu : %lu\n", session_id, (*c)->id); + + while (*c != NULL) + { + if ((*c)->id == session_id) + return (*c); + c = &(*c)->next; + } + + return ent; +} + +connection_t *connection_new() +{ + connection_t *con; + + con = (connection_t *)malloc(sizeof(connection_t)); + if (con == NULL) + { + DPRINTF("couldn't allocate a new connection\n"); + return NULL; + } + + con->bindings = NULL; + con->data_fd = -1; + con->ctrl_fd = -1; + + /* connections need a unique session id. + * - this approach probably gets fixed later, but for the moment + * is unique, and clearly identifies a connection. + */ + con->id = (unsigned long)con; + + /* add it to the connection list */ + con->next = connection_list; + connection_list = con; + + return (con); +} + +void connection_free(connection_t *con) +{ + /* first free all subscribed bindings: */ + + while (con->bindings != NULL) + xcs_unbind(con, con->bindings->key.port, con->bindings->key.type); + + /* now free the connection. */ + free(con); +} + +int connection_add_binding(connection_t *con, binding_key_t *key) +{ + binding_key_ent_t *key_ent; + + key_ent = (binding_key_ent_t *)malloc(sizeof(binding_key_ent_t)); + if (key_ent == NULL) + { + DPRINTF("couldn't alloc key in connection_add_binding\n"); + return -1; + } + + key_ent->key = *key; + key_ent->next = con->bindings; + con->bindings = key_ent; + + return 0; +} + +int connection_remove_binding(connection_t *con, binding_key_t *key) +{ + binding_key_ent_t *key_ent; + binding_key_ent_t **curs = &con->bindings; + + while ((*curs != NULL) && (!BINDING_KEYS_EQUAL(&(*curs)->key, key))) + curs = &(*curs)->next; + + if (*curs != NULL) { + key_ent = *curs; + *curs = (*curs)->next; + free(key_ent); + } + + return 0; +} + + +int connection_has_binding(connection_t *con, binding_key_t *key) +{ + binding_key_ent_t *ent; + int ret = 0; + + ent = con->bindings; + + while (ent != NULL) + { + if (BINDING_KEYS_EQUAL(key, &ent->key)) + { + ret = 1; + break; + } + ent = ent->next; + } + + return ret; +} + + +void gc_connection_list(void) +{ + connection_t **c, *ent = NULL; + struct timeval now, delta; + + c = &connection_list; + gettimeofday(&now, NULL); + + while ( *c != NULL ) + { + if ( !CONNECTED(*c) ) + { + timersub(&now, &(*c)->disconnect_time, &delta); + if ( delta.tv_sec >= XCS_SESSION_TIMEOUT ) + { + DPRINTF(" : Freeing connection %lu after %lds\n", + (*c)->id, delta.tv_sec); + ent = *c; + *c = (*c)->next; + connection_free(ent); + continue; + } + } + c = &(*c)->next; + } +} diff --git a/tools/xcs/ctrl_interface.c b/tools/xcs/ctrl_interface.c new file mode 100644 index 0000000000..0896910cb4 --- /dev/null +++ b/tools/xcs/ctrl_interface.c @@ -0,0 +1,269 @@ +/* control_interface.c + * + * Interfaces to control message rings to VMs. + * + * Most of this is directly based on the original xu interface to python + * written by Keir Fraser. + * + * (c) 2004, Andrew Warfield + * + */ + +#include +#include +#include +#include +#include +#include +#include "xcs.h" + +static int xc_handle = -1; + +/* Called at start-of-day when using the control channel interface. */ +int ctrl_chan_init(void) +{ + if ( (xc_handle = xc_interface_open()) == -1 ) + { + DPRINTF("Could not open Xen control interface"); + return -1; + } + + return 0; +} + +static control_if_t *map_control_interface(int fd, unsigned long pfn, + u32 dom) +{ + char *vaddr = xc_map_foreign_range( fd, dom, PAGE_SIZE, + PROT_READ|PROT_WRITE, pfn ); + if ( vaddr == NULL ) + return NULL; + return (control_if_t *)(vaddr + 2048); +} + +static void unmap_control_interface(int fd, control_if_t *c) +{ + char *vaddr = (char *)c - 2048; + (void)munmap(vaddr, PAGE_SIZE); +} + +int ctrl_chan_notify(control_channel_t *cc) +{ + return xc_evtchn_send(xc_handle, cc->local_port); +} + +int ctrl_chan_read_request(control_channel_t *cc, xcs_control_msg_t *dmsg) +{ + control_msg_t *smsg; + RING_IDX c = cc->tx_ring.req_cons; + + if ( !RING_HAS_UNCONSUMED_REQUESTS(CTRL_RING, &cc->tx_ring) ) + { + DPRINTF("no request to read\n"); + return -1; + } + + rmb(); /* make sure we see the data associated with the request */ + smsg = RING_GET_REQUEST(CTRL_RING, &cc->tx_ring, c); + memcpy(&dmsg->msg, smsg, sizeof(*smsg)); + if ( dmsg->msg.length > sizeof(dmsg->msg.msg) ) + dmsg->msg.length = sizeof(dmsg->msg.msg); + cc->tx_ring.req_cons++; + return 0; +} + +int ctrl_chan_write_request(control_channel_t *cc, + xcs_control_msg_t *smsg) +{ + control_msg_t *dmsg; + RING_IDX p = cc->rx_ring.req_prod_pvt; + + if ( RING_FULL(CTRL_RING, &cc->rx_ring) ) + { + DPRINTF("no space to write request"); + return -ENOSPC; + } + + dmsg = RING_GET_REQUEST(CTRL_RING, &cc->rx_ring, p); + memcpy(dmsg, &smsg->msg, sizeof(*dmsg)); + + wmb(); + cc->rx_ring.req_prod_pvt++; + RING_PUSH_REQUESTS(CTRL_RING, &cc->rx_ring); + + return 0; +} + +int ctrl_chan_read_response(control_channel_t *cc, xcs_control_msg_t *dmsg) +{ + control_msg_t *smsg; + RING_IDX c = cc->rx_ring.rsp_cons; + + if ( !RING_HAS_UNCONSUMED_RESPONSES(CTRL_RING, &cc->rx_ring) ) + { + DPRINTF("no response to read"); + return -1; + } + + rmb(); /* make sure we see the data associated with the request */ + smsg = RING_GET_RESPONSE(CTRL_RING, &cc->rx_ring, c); + memcpy(&dmsg->msg, smsg, sizeof(*smsg)); + if ( dmsg->msg.length > sizeof(dmsg->msg.msg) ) + dmsg->msg.length = sizeof(dmsg->msg.msg); + cc->rx_ring.rsp_cons++; + return 0; +} + +int ctrl_chan_write_response(control_channel_t *cc, + xcs_control_msg_t *smsg) +{ + control_msg_t *dmsg; + RING_IDX p = cc->tx_ring.rsp_prod_pvt; + + /* akw: if the ring is synchronous, you should never need this test! */ + /* (but it was in the original code... ) */ + if ( cc->tx_ring.req_cons == cc->tx_ring.rsp_prod_pvt ) + { + DPRINTF("no space to write response"); + return -ENOSPC; + } + + dmsg = RING_GET_RESPONSE(CTRL_RING, &cc->tx_ring, p); + memcpy(dmsg, &smsg->msg, sizeof(*dmsg)); + + wmb(); + cc->tx_ring.rsp_prod_pvt++; + RING_PUSH_RESPONSES(CTRL_RING, &cc->tx_ring); + + return 0; +} + +int ctrl_chan_request_to_read(control_channel_t *cc) +{ + return (RING_HAS_UNCONSUMED_REQUESTS(CTRL_RING, &cc->tx_ring)); +} + +int ctrl_chan_space_to_write_request(control_channel_t *cc) +{ + return (!(RING_FULL(CTRL_RING, &cc->rx_ring))); +} + +int ctrl_chan_response_to_read(control_channel_t *cc) +{ + return (RING_HAS_UNCONSUMED_RESPONSES(CTRL_RING, &cc->rx_ring)); +} + +int ctrl_chan_space_to_write_response(control_channel_t *cc) +{ + /* again, there is something fishy here. */ + return ( cc->tx_ring.req_cons != cc->tx_ring.rsp_prod_pvt ); +} + +int ctrl_chan_connect(control_channel_t *cc) +{ + xc_dominfo_t info; + + if ( cc->connected ) + { + return 0; + } + + if ( (xc_domain_getinfo(xc_handle, cc->remote_dom, 1, &info) != 1) || + (info.domid != cc->remote_dom) ) + { + DPRINTF("Failed to obtain domain status"); + return -1; + } + + cc->interface = + map_control_interface(xc_handle, info.shared_info_frame, + cc->remote_dom); + + if ( cc->interface == NULL ) + { + DPRINTF("Failed to map domain control interface"); + return -1; + } + + /* Synchronise ring indexes. */ + BACK_RING_ATTACH(CTRL_RING, &cc->tx_ring, &cc->interface->tx_ring); + FRONT_RING_ATTACH(CTRL_RING, &cc->rx_ring, &cc->interface->rx_ring); + + cc->connected = 1; + + return 0; +} + +void ctrl_chan_disconnect(control_channel_t *cc) +{ + if ( cc->connected ) + unmap_control_interface(xc_handle, cc->interface); + cc->connected = 0; +} + + +control_channel_t *ctrl_chan_new(u32 dom, int local_port, int remote_port) +{ + control_channel_t *cc; + + cc = (control_channel_t *)malloc(sizeof(control_channel_t)); + if ( cc == NULL ) return NULL; + + cc->connected = 0; + cc->remote_dom = dom; + + if ( dom == 0 ) + { + /* + * The control-interface event channel for DOM0 is already set up. + * We use an ioctl to discover the port at our end of the channel. + */ + local_port = ioctl(xc_handle, IOCTL_PRIVCMD_INITDOMAIN_EVTCHN, + NULL); + remote_port = -1; /* We don't need the remote end of the DOM0 link. */ + if ( local_port < 0 ) + { + DPRINTF("Could not open channel to DOM0"); + goto fail; + } + } + else if ( xc_evtchn_bind_interdomain(xc_handle, + DOMID_SELF, dom, + &local_port, &remote_port) != 0 ) + { + DPRINTF("Could not open channel to domain"); + goto fail; + } + + cc->local_port = local_port; + cc->remote_port = remote_port; + + if ( ctrl_chan_connect(cc) != 0 ) + goto fail; + + return cc; + + fail: + if ( dom != 0 ) + (void)xc_evtchn_close(xc_handle, DOMID_SELF, local_port); + + free(cc); + + return NULL; +} + +void ctrl_chan_free(control_channel_t *cc) +{ + ctrl_chan_disconnect(cc); + if ( cc->remote_dom != 0 ) + (void)xc_evtchn_close(xc_handle, DOMID_SELF, cc->local_port); + free(cc); +} + + +/* other libxc commands: */ + +int ctrl_chan_bind_virq(int virq, int *port) +{ + return xc_evtchn_bind_virq(xc_handle, virq, port); +} diff --git a/tools/xcs/evtchn.c b/tools/xcs/evtchn.c new file mode 100644 index 0000000000..a96036db37 --- /dev/null +++ b/tools/xcs/evtchn.c @@ -0,0 +1,108 @@ +/* evtchn.c + * + * Interfaces to event channel driver. + * + * Most of this is directly based on the original xu interface to python + * written by Keir Fraser. + * + * (c) 2004, Andrew Warfield + * + */ + +#include +#include +#include +#include +#include /* XOPEN drops makedev, this gets it back. */ +#include +#include +#include +#include "xcs.h" + +static int evtchn_fd = -1; + +/* NB. The following should be kept in sync with the kernel's evtchn driver. */ +#define EVTCHN_DEV_NAME "/dev/xen/evtchn" +#define EVTCHN_DEV_MAJOR 10 +#define EVTCHN_DEV_MINOR 201 +/* /dev/xen/evtchn ioctls: */ +/* EVTCHN_RESET: Clear and reinit the event buffer. Clear error condition. */ +#define EVTCHN_RESET _IO('E', 1) +/* EVTCHN_BIND: Bind to teh specified event-channel port. */ +#define EVTCHN_BIND _IO('E', 2) +/* EVTCHN_UNBIND: Unbind from the specified event-channel port. */ +#define EVTCHN_UNBIND _IO('E', 3) + +int evtchn_read() +{ + u16 v; + int bytes; + + while ( (bytes = read(evtchn_fd, &v, sizeof(v))) == -1 ) + { + if ( errno == EINTR ) + continue; + /* EAGAIN was cased to return 'None' in the python version... */ + return -errno; + } + + if ( bytes == sizeof(v) ) + return v; + + /* bad return */ + return -1; +} + +void evtchn_unmask(u16 idx) +{ + (void)write(evtchn_fd, &idx, sizeof(idx)); +} + +int evtchn_bind(int idx) +{ + if ( ioctl(evtchn_fd, EVTCHN_BIND, idx) != 0 ) + return -errno; + + return 0; +} + +int evtchn_unbind(int idx) +{ + if ( ioctl(evtchn_fd, EVTCHN_UNBIND, idx) != 0 ) + return -errno; + + return 0; +} + +int evtchn_open(void) +{ + struct stat st; + + /* Make sure any existing device file links to correct device. */ + if ( (lstat(EVTCHN_DEV_NAME, &st) != 0) || + !S_ISCHR(st.st_mode) || + (st.st_rdev != makedev(EVTCHN_DEV_MAJOR, EVTCHN_DEV_MINOR)) ) + (void)unlink(EVTCHN_DEV_NAME); + + reopen: + evtchn_fd = open(EVTCHN_DEV_NAME, O_NONBLOCK|O_RDWR); + if ( evtchn_fd == -1 ) + { + if ( (errno == ENOENT) && + ((mkdir("/dev/xen", 0755) == 0) || (errno == EEXIST)) && + (mknod(EVTCHN_DEV_NAME, S_IFCHR|0600, + makedev(EVTCHN_DEV_MAJOR,EVTCHN_DEV_MINOR)) == 0) ) + goto reopen; + return -errno; + } + /*set_cloexec(evtchn_fd); -- no longer required*/ +printf("Eventchan_fd is %d\n", evtchn_fd); + return evtchn_fd; +} + +void evtchn_close() +{ + (void)close(evtchn_fd); + evtchn_fd = -1; +} + diff --git a/tools/xcs/xcs.c b/tools/xcs/xcs.c new file mode 100644 index 0000000000..785614399f --- /dev/null +++ b/tools/xcs/xcs.c @@ -0,0 +1,833 @@ +/* xcs.c + * + * xcs - Xen Control Switch + * + * Copyright (c) 2004, Andrew Warfield + */ + +/* + + Things we need to select on in xcs: + + 1. Events arriving on /dev/evtchn + + These will kick a function to read everything off the fd, and scan the + associated control message rings, resulting in notifications sent on + data channels to connected clients. + + 2. New TCP connections on XCS_PORT. + + These will either be control (intially) or associated data connections. + + Control connections will instantiate or rebind to an existing connnection + struct. The control channel is used to configure what events will be + received on an associated data channel. These two channels are split + out because the control channel is synchronous, all messages will return + a result from XCS. The data channel is effectively asynchronous, events + may arrive in the middle of a control message exchange. Additionally, + Having two TCP connections allows the client side to have a blocking + listen loop for data messages, while independently interacting on the + control channel at other places in the code. + + Data connections attach to an existing control struct, using a session + id that is passed during the control connect. There is currently a + one-to-one relationship between data and control channels, but there + could just as easily be many data channels, if there were a set of + clients with identical interests, or if you wanted to trace an existing + client's data traffic. + + 3. Messages arriving on open TCP connections. + There are three types of open connections: + + 3a. Messages arriving on open control channel file descriptors. + + [description of the control protocol here] + + 3b. Messages arriving on open data channel file descriptors. + + [description of the data protocol here] + + 3c. Messages arriving on (new) unbound connections. + + A connection must issue a XCS_CONNECT message to specify what + it is, after which the connection is moved into one of the above + two groups. + + Additionally, we need a periodic timer to do housekeeping. + + 4. Every XCS_GC_INTERVAL seconds, we need to clean up outstanding state. + Specifically, we garbage collect any sessions (connection_t structs) + that have been unconnected for a period of time (XCS_SESSION_TIMEOUT), + and close any connections that have been openned, but not connected + as a control or data connection (XCS_UFD_TIMEOUT). + +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "xcs.h" + +#undef fd_max +#define fd_max(x,y) ((x) > (y) ? (x) : (y)) + +/* ------[ Control channel interfaces ]------------------------------------*/ + +static control_channel_t *cc_list[NR_EVENT_CHANNELS]; +static int dom_to_port[MAX_DOMS]; /* This should not be a fixed-size array.*/ + +static void init_interfaces(void) +{ + int i; + + for (i = 0; i < MAX_DOMS; i++) + dom_to_port[i] = -1; + memset(cc_list, 0, sizeof cc_list); +} + +static control_channel_t *add_interface(u32 dom, int local_port, + int remote_port) +{ + control_channel_t *cc, *oldcc; + int ret; + + if (cc_list[dom_to_port[dom]] != NULL) + { + return(cc_list[dom_to_port[dom]]); + } + + if (cc_list[local_port] == NULL) + { + cc = ctrl_chan_new(dom, local_port, remote_port); + } + + if (cc == NULL) + return NULL; + + DPRINTF("added a new interface: dom: %u (l:%d,r:%d): %p\n", + dom, local_port, remote_port, cc); + DPRINTF("added a new interface: dom: %u (l:%d,r:%d): %p\n", + dom, cc->local_port, cc->remote_port, cc); + + if ((ret = evtchn_bind(cc->local_port)) != 0) + { + DPRINTF("Got control interface, but couldn't bind evtchan!(%d)\n", ret); + ctrl_chan_free(cc); + return NULL; + } + + if ( cc_list[cc->local_port] != NULL ) + { + oldcc = cc_list[cc->local_port]; + + if ((oldcc->remote_dom != cc->remote_dom) || + (oldcc->remote_port != cc->remote_port)) + { + DPRINTF("CC conflict! (port: %d, old dom: %u, new dom: %u)\n", + cc->local_port, oldcc->remote_dom, cc->remote_dom); + dom_to_port[oldcc->remote_dom] = -1; + ctrl_chan_free(cc_list[cc->local_port]); + } + } + + cc_list[cc->local_port] = cc; + dom_to_port[cc->remote_dom] = cc->local_port; + cc->type = CC_TYPE_INTERDOMAIN; + cc->ref_count = 0; + return cc; +} + +control_channel_t *add_virq(int virq) +{ + control_channel_t *cc; + int virq_port; + + if (ctrl_chan_bind_virq(virq, &virq_port) == -1) + return NULL; + + if ((cc_list[virq_port] != NULL) && + (cc_list[virq_port]->type != CC_TYPE_VIRQ)) + return NULL; + + if ((cc_list[virq_port] != NULL) && + (cc_list[virq_port]->type == CC_TYPE_VIRQ)) + return cc_list[virq_port]; + + cc = (control_channel_t *)malloc(sizeof(control_channel_t)); + if ( cc == NULL ) return NULL; + + cc->type = CC_TYPE_VIRQ; + cc->local_port = virq_port; + cc->virq = virq; + + return cc; +} + +void get_interface(control_channel_t *cc) +{ + if (cc != NULL) + cc->ref_count++; +} + +void put_interface(control_channel_t *cc) +{ + if (cc != NULL) + { + cc->ref_count--; + if (cc->ref_count <= 0) + { + DPRINTF("Freeing cc on port %d.\n", cc->local_port); + (void)evtchn_unbind(cc->local_port); + ctrl_chan_free(cc); + } + } +} + +/* ------[ Simple helpers ]------------------------------------------------*/ + +/* listen_socket() is straight from paul sheer's useful select_tut manpage. */ +static int listen_socket (int listen_port) +{ + struct sockaddr_in a; + int s; + int yes; + + if ((s = socket (AF_INET, SOCK_STREAM, 0)) < 0) + { + perror ("socket"); + return -1; + } + + yes = 1; + if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, + (char *) &yes, sizeof (yes)) < 0) + { + perror ("setsockopt"); + close (s); + return -1; + } + + memset (&a, 0, sizeof (a)); + a.sin_port = htons (listen_port); + a.sin_family = AF_INET; + if (bind(s, (struct sockaddr *) &a, sizeof (a)) < 0) + { + perror ("bind"); + close (s); + return -1; + } + printf ("accepting connections on port %d\n", (int) listen_port); + listen (s, 10); + return s; +} + +/* ------[ Message handlers ]----------------------------------------------*/ + +#define NO_CHANGE 0 +#define CONNECTED 1 +#define DISCONNECTED 2 +int handle_connect_msg( xcs_msg_t *msg, int fd ) +{ + xcs_connect_msg_t *cmsg = &msg->u.connect; + connection_t *con; + int ret = NO_CHANGE; + + switch (msg->type) + { + case XCS_CONNECT_CTRL: + { + if ( cmsg->session_id == 0 ) + { + con = connection_new(); + if ( con == NULL) + { + msg->result = XCS_RSLT_FAILED; + break; + } + msg->result = XCS_RSLT_OK; + cmsg->session_id = con->id; + con->ctrl_fd = fd; + ret = CONNECTED; + DPRINTF("New control connection\n"); + break; + } + + con = get_con_by_session(cmsg->session_id); + if ( con == NULL ) + { + msg->result = XCS_RSLT_BADSESSION; + break; + } + if ( con->ctrl_fd != -1 ) + { + msg->result = XCS_RSLT_CONINUSE; + break; + } + con->ctrl_fd = fd; + msg->result = XCS_RSLT_OK; + ret = CONNECTED; + DPRINTF("Rebound to control connection\n"); + break; + } + case XCS_CONNECT_DATA: + { + con = get_con_by_session(cmsg->session_id); + if ( con == NULL ) + { + msg->result = XCS_RSLT_BADSESSION; + break; + } + if ( con->data_fd != -1 ) + { + msg->result = XCS_RSLT_CONINUSE; + break; + } + con->data_fd = fd; + msg->result = XCS_RSLT_OK; + ret = CONNECTED; + DPRINTF("Attached data connection\n"); + break; + + } + case XCS_CONNECT_BYE: + { + close ( fd ); + ret = DISCONNECTED; + break; + } + } + + return ret; +} + +int handle_control_message( connection_t *con, xcs_msg_t *msg ) +{ + int ret; + int reply_needed = 1; + + DPRINTF("Got message, type %u.\n", msg->type); + + switch (msg->type) + { + case XCS_MSG_BIND: + { + xcs_bind_msg_t *bmsg = &msg->u.bind; + + if ( ! BIND_MSG_VALID(bmsg) ) + { + msg->result = XCS_RSLT_BADREQUEST; + break; + } + + ret = xcs_bind(con, bmsg->port, bmsg->type); + if (ret == 0) { + msg->result = XCS_RSLT_OK; + } else { + msg->result = XCS_RSLT_FAILED; + } + break; + } + case XCS_MSG_UNBIND: + { + xcs_bind_msg_t *bmsg = &msg->u.bind; + + if ( ! BIND_MSG_VALID(bmsg) ) + { + msg->result = XCS_RSLT_BADREQUEST; + break; + } + + ret = xcs_unbind(con, bmsg->port, bmsg->type); + if (ret == 0) { + msg->result = XCS_RSLT_OK; + } else { + msg->result = XCS_RSLT_FAILED; + } + break; + } + case XCS_VIRQ_BIND: + { + control_channel_t *cc; + xcs_virq_msg_t *vmsg = &msg->u.virq; + if ( ! VIRQ_MSG_VALID(vmsg) ) + { + msg->result = XCS_RSLT_BADREQUEST; + break; + } + + cc = add_virq(vmsg->virq); + if (cc == NULL) + { + msg->result = XCS_RSLT_FAILED; + break; + } + ret = xcs_bind(con, cc->local_port, TYPE_VIRQ); + if (ret == 0) { + vmsg->port = cc->local_port; + msg->result = XCS_RSLT_OK; + } else { + msg->result = XCS_RSLT_FAILED; + } + break; + } + + case XCS_CIF_NEW_CC: + { + control_channel_t *cc; + xcs_interface_msg_t *imsg = &msg->u.interface; + + if ( ! INTERFACE_MSG_VALID(imsg) ) + { + msg->result = XCS_RSLT_BADREQUEST; + break; + } + + cc = add_interface(imsg->dom, imsg->local_port, imsg->remote_port); + if (cc != NULL) { + get_interface(cc); + msg->result = XCS_RSLT_OK; + imsg->local_port = cc->local_port; + imsg->remote_port = cc->remote_port; + } else { + msg->result = XCS_RSLT_FAILED; + } + break; + } + + case XCS_CIF_FREE_CC: + { + control_channel_t *cc; + xcs_interface_msg_t *imsg = &msg->u.interface; + + if ( ! INTERFACE_MSG_VALID(imsg) ) + { + msg->result = XCS_RSLT_BADREQUEST; + break; + } + + cc = add_interface(imsg->dom, imsg->local_port, imsg->remote_port); + if (cc != NULL) { + put_interface(cc); + } + msg->result = XCS_RSLT_OK; + break; + } + } + return reply_needed; +} + +void handle_data_message( connection_t *con, xcs_msg_t *msg ) +{ + control_channel_t *cc; + xcs_control_msg_t *cmsg = &msg->u.control; + int port; + + switch (msg->type) + { + case XCS_REQUEST: + if ( cmsg->remote_dom > MAX_DOMS ) + break; + + port = dom_to_port[cmsg->remote_dom]; + if (port == -1) break; + cc = cc_list[port]; + if ((cc != NULL) && ( cc->type == CC_TYPE_INTERDOMAIN )) + { + DPRINTF("DN:REQ: dom:%d port: %d type: %d\n", + cc->remote_dom, cc->local_port, + cmsg->msg.type); + ctrl_chan_write_request(cc, cmsg); + ctrl_chan_notify(cc); + } else { + DPRINTF("tried to send a REQ to a null cc\n."); + } + break; + + case XCS_RESPONSE: + if ( cmsg->remote_dom > MAX_DOMS ) + break; + + port = dom_to_port[cmsg->remote_dom]; + if (port == -1) break; + cc = cc_list[port]; + if ((cc != NULL) && ( cc->type == CC_TYPE_INTERDOMAIN )) + { + DPRINTF("DN:RSP: dom:%d port: %d type: %d\n", + cc->remote_dom, cc->local_port, + cmsg->msg.type); + ctrl_chan_write_response(cc, cmsg); + ctrl_chan_notify(cc); + } + break; + + case XCS_VIRQ: + if ( !(PORT_VALID(cmsg->local_port)) ) + break; + + cc = cc_list[cmsg->local_port]; + + if ((cc != NULL) && ( cc->type == CC_TYPE_VIRQ )) + { + DPRINTF("DN:VIRQ: virq: %d port: %d\n", + cc->virq, cc->local_port); + ctrl_chan_notify(cc); + } + break; + } +} + +/* ------[ Control interface handler ]-------------------------------------*/ + +/* passed as a function pointer to the lookup. */ +void send_kmsg(connection_t *c, void *arg) +{ + xcs_msg_t *msg = (xcs_msg_t *)arg; + + DPRINTF(" -> CONNECTION %d\n", c->data_fd); + if (c->data_fd > 0) + { + send(c->data_fd, msg, sizeof(xcs_msg_t), 0); + } +} + +int handle_ctrl_if(void) +{ + control_channel_t *cc; + control_msg_t *msg; + xcs_msg_t kmsg; + int chan, ret; + + DPRINTF("Event thread kicked!\n"); +again: + while ((chan = evtchn_read()) > 0) + { + evtchn_unmask(chan); + cc = cc_list[chan]; + if (cc_list[chan] == NULL) { + DPRINTF("event from unknown channel (%d)\n", chan); + continue; + } + + if ( cc_list[chan]->type == CC_TYPE_VIRQ ) + { + DPRINTF("UP:VIRQ: virq:%d port: %d\n", + cc->virq, cc->local_port); + kmsg.type = XCS_VIRQ; + kmsg.u.control.local_port = cc->local_port; + xcs_lookup(cc->local_port, TYPE_VIRQ, send_kmsg, &kmsg); + continue; + } + + while (ctrl_chan_request_to_read(cc)) + { + msg = &kmsg.u.control.msg; + kmsg.type = XCS_REQUEST; + kmsg.u.control.remote_dom = cc->remote_dom; + kmsg.u.control.local_port = cc->local_port; + ret = ctrl_chan_read_request(cc, &kmsg.u.control); + DPRINTF("UP:REQ: dom:%d port: %d type: %d len: %d\n", + cc->remote_dom, cc->local_port, + msg->type, msg->length); + if (ret == 0) + xcs_lookup(cc->local_port, msg->type, send_kmsg, &kmsg); + } + + while (ctrl_chan_response_to_read(cc)) + { + msg = &kmsg.u.control.msg; + kmsg.type = XCS_RESPONSE; + kmsg.u.control.remote_dom = cc->remote_dom; + kmsg.u.control.local_port = cc->local_port; + ret = ctrl_chan_read_response(cc, &kmsg.u.control); + DPRINTF("UP:RSP: dom:%d port: %d type: %d len: %d\n", + cc->remote_dom, cc->local_port, + msg->type, msg->length); + if (ret == 0) + xcs_lookup(cc->local_port, msg->type, send_kmsg, &kmsg); + } + } + + if (chan == -EINTR) + goto again; + + return chan; +} + + +/* ------[ Main xcs code / big select loop ]-------------------------------*/ + + +typedef struct unbound_fd_st { + int fd; + struct timeval born; + struct unbound_fd_st *next; +} unbound_fd_t; + +/* This makes ufd point to the next entry in the list, so need to * + * break/continue if called while iterating. */ +void delete_ufd(unbound_fd_t **ufd) +{ + unbound_fd_t *del_ufd; + + del_ufd = *ufd; + *ufd = (*ufd)->next; + free( del_ufd ); +} + +void gc_ufd_list( unbound_fd_t **ufd ) +{ + struct timeval now, delta; + + gettimeofday(&now, NULL); + + while ( *ufd != NULL ) + { + timersub(&now, &(*ufd)->born, &delta); + if (delta.tv_sec > XCS_UFD_TIMEOUT) + { + DPRINTF("GC-UFD: closing fd: %d\n", (*ufd)->fd); + close((*ufd)->fd); + delete_ufd(ufd); + continue; + } + ufd = &(*ufd)->next; + } +} + +int main (int argc, char*argv[]) +{ + int listen_fd, evtchn_fd; + unbound_fd_t *unbound_fd_list = NULL, **ufd; + struct timeval timeout = { XCS_GC_INTERVAL, 0 }; + connection_t **con; + + /* Initialize xc and event connections. */ + if (ctrl_chan_init() != 0) + { + printf("Couldn't open conneciton to libxc.\n"); + exit(-1); + } + + if ((evtchn_fd = evtchn_open()) < 0) + { + printf("Couldn't open event channel driver interface.\n"); + exit(-1); + } + + /* Initialize control interfaces, bindings. */ + init_interfaces(); + init_bindings(); + + listen_fd = listen_socket(XCS_TCP_PORT); + + for (;;) + { + int n, ret; + fd_set rd, wr, er; + FD_ZERO ( &rd ); + FD_ZERO ( &wr ); + FD_ZERO ( &er ); + + /* TCP listen fd: */ + FD_SET ( listen_fd, &rd ); + n = fd_max ( n, listen_fd ); + + /* Evtchn fd: */ + FD_SET ( evtchn_fd, &rd ); + n = fd_max ( n, evtchn_fd ); + + /* unbound connection fds: */ + ufd = &unbound_fd_list; + while ((*ufd) != NULL) + { + FD_SET ( (*ufd)->fd, &rd ); + n = fd_max ( n, (*ufd)->fd ); + ufd = &(*ufd)->next; + } + + /* control and data fds: */ + con = &connection_list; + while ((*con) != NULL) + { + if ((*con)->ctrl_fd > 0) + { + FD_SET ( (*con)->ctrl_fd, &rd ); + n = fd_max ( n, (*con)->ctrl_fd ); + } + if ((*con)->data_fd > 0) + { + FD_SET ( (*con)->data_fd, &rd ); + n = fd_max ( n, (*con)->data_fd ); + } + con = &(*con)->next; + } + + ret = select ( n + 1, &rd, &wr, &er, &timeout ); + + if ( (timeout.tv_sec == 0) && (timeout.tv_usec == 0) ) + { + gc_ufd_list(&unbound_fd_list); + gc_connection_list(); + timeout.tv_sec = XCS_GC_INTERVAL; + } + + if ( (ret == -1) && (errno == EINTR) ) + continue; + if ( ret < 0 ) + { + perror ("select()"); + exit(-1); + } + + /* CASE 1: Events arriving on /dev/evtchn. */ + + if ( FD_ISSET (evtchn_fd, &rd )) + handle_ctrl_if(); + + /* CASE 2: New connection on the listen port. */ + if ( FD_ISSET ( listen_fd, &rd )) + { + struct sockaddr_in remote_addr; + int size; + memset (&remote_addr, 0, sizeof (remote_addr)); + size = sizeof remote_addr; + ret = accept(listen_fd, (struct sockaddr *)&remote_addr, &size); + if ( ret < 0 ) + { + perror("accept()"); + } else { + unbound_fd_t *new_ufd; + + new_ufd = (unbound_fd_t *)malloc(sizeof(*new_ufd)); + + if (new_ufd != NULL) + { + gettimeofday(&new_ufd->born, NULL); + new_ufd->fd = ret; + new_ufd->next = unbound_fd_list; + unbound_fd_list = new_ufd; + } else { + perror("malloc unbound connection"); + close(ret); + } + } + } + + /* CASE 3a: Handle messages on control connections. */ + + con = &connection_list; + while ( *con != NULL ) + { + if ( ((*con)->ctrl_fd > 0) && (FD_ISSET((*con)->ctrl_fd, &rd)) ) + { + xcs_msg_t msg; + memset (&msg, 0, sizeof(msg)); + ret = read( (*con)->ctrl_fd, &msg, sizeof(msg) ); + + if ( ret < 0 ) + { + perror("reading ctrl fd."); + } else if ( ret == 0 ) + { + DPRINTF("Control connection dropped.\n"); + close ( (*con)->ctrl_fd ); + (*con)->ctrl_fd = -1; + gettimeofday(&(*con)->disconnect_time, NULL); + } else + { + if ( ret != sizeof(msg) ) + { + DPRINTF("Unexpected frame size!\n"); + continue; + } + + ret = handle_control_message( *con, &msg ); + + if ( ret == 1 ) + send( (*con)->ctrl_fd, &msg, sizeof(msg), 0 ); + } + } + con = &(*con)->next; + } + + /* CASE 3b: Handle messages on data connections. */ + + con = &connection_list; + while ( *con != NULL ) + { + if ( ((*con)->data_fd > 0) && (FD_ISSET((*con)->data_fd, &rd)) ) + { + xcs_msg_t msg; + memset (&msg, 0, sizeof(msg)); + ret = read( (*con)->data_fd, &msg, sizeof(msg) ); + + if ( ret < 0 ) + { + perror("reading data fd."); + } else if ( ret == 0 ) + { + DPRINTF("Data connection dropped.\n"); + close ( (*con)->data_fd ); + (*con)->data_fd = -1; + gettimeofday(&(*con)->disconnect_time, NULL); + } else + { + if ( ret != sizeof(msg) ) + { + DPRINTF("Unexpected frame size!\n"); + continue; + } + + handle_data_message( *con, &msg ); + } + } + con = &(*con)->next; + } + + /* CASE 3c: Handle messages arriving on unbound connections. */ + ufd = &unbound_fd_list; + while ((*ufd) != NULL) + { + if ( FD_ISSET( (*ufd)->fd, &rd ) ) + { + xcs_msg_t msg; + memset (&msg, 0, sizeof(msg)); + ret = read( (*ufd)->fd, &msg, sizeof(msg) ); + + if ( ret == 0 ) + { + close ( (*ufd)->fd ); + delete_ufd(ufd); + continue; /* we just advanced ufd */ + } else { + if ( ret != sizeof(msg) ) + { + DPRINTF("Unexpected frame size!\n"); + continue; + } + + ret = handle_connect_msg( &msg, (*ufd)->fd ); + + if ( (ret == CONNECTED) || (ret == NO_CHANGE) ) + send( (*ufd)->fd, &msg, sizeof(msg), 0 ); + + if ( (ret = CONNECTED) || (ret = DISCONNECTED) ) + { + delete_ufd( ufd ); + continue; + } + } + } + ufd = &(*ufd)->next; + } + } +} + diff --git a/tools/xcs/xcs.h b/tools/xcs/xcs.h new file mode 100644 index 0000000000..545fc3d05f --- /dev/null +++ b/tools/xcs/xcs.h @@ -0,0 +1,155 @@ +/* xcs.h + * + * public interfaces for the control interface switch (xcs). + * + * (c) 2004, Andrew Warfield + * + */ + + +#ifndef __XCS_H__ +#define __XCS_H__ + +#include +#include +#include +#include +#include +#include +#include "xcs_proto.h" + +/* ------[ Debug macros ]--------------------------------------------------*/ + +#if 0 +#define DPRINTF(_f, _a...) printf ( _f , ## _a ) +#else +#define DPRINTF(_f, _a...) ((void)0) +#endif + +/* ------[ XCS-specific defines and types ]--------------------------------*/ + +#define MAX_DOMS 1024 +#define XCS_SESSION_TIMEOUT 10 /* (secs) disconnected session gc timeout */ +#define XCS_UFD_TIMEOUT 5 /* how long can connections be unbound? */ +#define XCS_GC_INTERVAL 5 /* How often to run gc handlers. */ + + +/* ------[ Other required defines ]----------------------------------------*/ + +/* Size of a machine page frame. */ +#define PAGE_SIZE 4096 + +#if defined(__i386__) +#define rmb() __asm__ __volatile__ ( "lock; addl $0,0(%%esp)" : : : "memory" ) +#define wmb() __asm__ __volatile__ ( "" : : : "memory" ) +#else +#error "Define barriers" +#endif + +#ifndef timersub /* XOPEN and __BSD don't cooperate well... */ +#define timersub(a, b, result) \ + do { \ + (result)->tv_sec = (a)->tv_sec - (b)->tv_sec; \ + (result)->tv_usec = (a)->tv_usec - (b)->tv_usec; \ + if ((result)->tv_usec < 0) { \ + --(result)->tv_sec; \ + (result)->tv_usec += 1000000; \ + } \ + } while (0) +#endif /*timersub*/ + +/* ------[ Bindings Interface ]--------------------------------------------*/ + +/*forward declare connection_t */ +typedef struct connection_st connection_t; + +typedef struct { + int port; + u16 type; +} binding_key_t; + +typedef struct binding_key_ent_st { + binding_key_t key; + struct binding_key_ent_st *next; +} binding_key_ent_t; + +#define BINDING_KEYS_EQUAL(_k1, _k2) \ + (((_k1)->port == (_k2)->port) && ((_k1)->type == (_k2)->type)) + +int xcs_bind(connection_t *con, int port, u16 type); +int xcs_unbind(connection_t *con, int port, u16 type); +void xcs_lookup(int port, u16 type, void (*f)(connection_t *, void *), + void *arg); +void init_bindings(void); + +/* ------[ Connection Interface ]------------------------------------------*/ + +struct connection_st { + unsigned long id; /* Unique session id */ + int ctrl_fd; /* TCP descriptors */ + int data_fd; /* */ + binding_key_ent_t *bindings; /* List of bindings */ + connection_t *next; /* Linked list of connections */ + struct timeval disconnect_time; /* " " */ +}; /* previously typedefed as connection_t */ + + +extern connection_t *connection_list; + +connection_t *get_con_by_session(unsigned long session_id); +connection_t *connection_new(); +void connection_free(connection_t *con); +int connection_add_binding(connection_t *con, binding_key_t *key); +int connection_remove_binding(connection_t *con, binding_key_t *key); +int connection_has_binding(connection_t *con, binding_key_t *key); +void gc_connection_list(void); + +/* ------[ Control Channel Interfaces ]------------------------------------*/ + +typedef struct { + int connected; + int ref_count; + int type; + u32 remote_dom; + int local_port; + int remote_port; + control_if_t *interface; + ctrl_back_ring_t tx_ring; + ctrl_front_ring_t rx_ring; + int virq; +} control_channel_t; + +/* cc types that we care about */ +#define CC_TYPE_INTERDOMAIN 0 +#define CC_TYPE_VIRQ 1 + +control_channel_t + *ctrl_chan_new(u32 dom, int local_port, int remote_port); +void ctrl_chan_free(control_channel_t *cc); +int ctrl_chan_init(void); +int ctrl_chan_notify(control_channel_t *cc); +int ctrl_chan_read_request(control_channel_t *cc, xcs_control_msg_t *); +int ctrl_chan_write_request(control_channel_t *cc, + xcs_control_msg_t *smsg); +int ctrl_chan_read_response(control_channel_t *cc, xcs_control_msg_t *); +int ctrl_chan_write_response(control_channel_t *cc, + xcs_control_msg_t *smsg); +int ctrl_chan_request_to_read(control_channel_t *cc); +int ctrl_chan_space_to_write_request(control_channel_t *cc); +int ctrl_chan_response_to_read(control_channel_t *cc); +int ctrl_chan_space_to_write_response(control_channel_t *cc); +int ctrl_chan_connect(control_channel_t *cc); +void ctrl_chan_disconnect(control_channel_t *cc); +int ctrl_chan_bind_virq(int virq, int *port); + +/* ------[ Event notification interfaces ]---------------------------------*/ + + +int evtchn_open(void); +void evtchn_close(); +int evtchn_bind(int idx); +int evtchn_unbind(int idx); +void evtchn_unmask(u16 idx); +int evtchn_read(); + +#endif /* __XCS_H__ */ diff --git a/tools/xcs/xcs_proto.h b/tools/xcs/xcs_proto.h new file mode 100644 index 0000000000..ea227c2ff7 --- /dev/null +++ b/tools/xcs/xcs_proto.h @@ -0,0 +1,101 @@ +/* xcs_proto.h + * + * protocol interfaces for the control interface switch (xcs). + * + * (c) 2004, Andrew Warfield + * + */ + +#ifndef __XCS_PROTO_H__ +#define __XCS_PROTO_H__ + +#define XCS_TCP_PORT 1633 + +/* xcs message types: */ +#define XCS_CONNECT_CTRL 0 /* This is a control connection. */ +#define XCS_CONNECT_DATA 1 /* This is a data connection. */ +#define XCS_CONNECT_BYE 2 /* Terminate a session. */ +#define XCS_MSG_BIND 3 /* Register for a message type. */ +#define XCS_MSG_UNBIND 4 /* Unregister for a message type. */ +#define XCS_VIRQ_BIND 5 /* Register for a virq. */ +#define XCS_MSG_WRITELOCK 6 /* Writelock a (dom,type) pair. */ +#define XCS_CIF_NEW_CC 7 /* Create a new control channel. */ +#define XCS_CIF_FREE_CC 8 /* Create a new control channel. */ +#define XCS_REQUEST 9 /* This is a request message. */ +#define XCS_RESPONSE 10 /* this is a response Message. */ +#define XCS_VIRQ 11 /* this is a virq notification. */ + +/* xcs result values: */ +#define XCS_RSLT_OK 0 +#define XCS_RSLT_FAILED 1 /* something bad happened. */ +#define XCS_RSLT_ARECONNECTED 2 /* attempt to over connect. */ +#define XCS_RSLT_BADSESSION 3 /* request for unknown session id. */ +#define XCS_RSLT_NOSESSION 4 /* tried to do something before NEW. */ +#define XCS_RSLT_CONINUSE 5 /* Requested connection is taken. */ +#define XCS_RSLT_BADREQUEST 6 /* Request message didn't validate. */ + +/* Binding wildcards */ +#define PORT_WILDCARD 0xefffffff +#define TYPE_WILDCARD 0xffff +#define TYPE_VIRQ 0xfffe + +typedef struct { + u32 session_id; +} xcs_connect_msg_t; + +typedef struct { + int port; + u16 type; +} xcs_bind_msg_t; + +typedef struct { + int port; + u16 virq; +} xcs_virq_msg_t; + +typedef struct { + u32 dom; + int local_port; + int remote_port; +} xcs_interface_msg_t; + +typedef struct { + u32 remote_dom; + int local_port; + control_msg_t msg; +} xcs_control_msg_t; + +typedef struct { + u32 type; + u32 result; + union { + xcs_connect_msg_t connect; /* These are xcs ctrl message types */ + xcs_bind_msg_t bind; + xcs_virq_msg_t virq; + xcs_interface_msg_t interface; + + xcs_control_msg_t control; /* These are xcs data message types */ + } PACKED u; +} xcs_msg_t; + +/* message validation macros. */ +#define PORT_VALID(_p) \ + ( (((_p) >= 0) && ((_p) < NR_EVENT_CHANNELS)) \ + || ((_p) == PORT_WILDCARD) ) + +#define TYPE_VALID(_t) \ + ( ((_t) < 256) \ + || ((_t) == TYPE_VIRQ) \ + || ((_t) == TYPE_WILDCARD) ) + +#define BIND_MSG_VALID(_b) \ + ( PORT_VALID((_b)->port) && TYPE_VALID((_b)->type) ) + +/* Port is overwritten, and we don't currently validate the requested virq. */ +#define VIRQ_MSG_VALID(_v) ( 1 ) + +/* Interfaces may return with ports of -1, but may not be requested as such */ +#define INTERFACE_MSG_VALID(_i) \ + ( PORT_VALID((_i)->local_port) && PORT_VALID((_i)->remote_port) ) + +#endif /* __XCS_PROTO_H__ */ diff --git a/tools/xcs/xcsdump.c b/tools/xcs/xcsdump.c new file mode 100644 index 0000000000..dcfd2c9119 --- /dev/null +++ b/tools/xcs/xcsdump.c @@ -0,0 +1,182 @@ +/* xcsdump.c + * + * little tool to sniff control messages. + * + * Copyright (c) 2004, Andrew Warfield + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "xcs_proto.h" +#include "xcs.h" + +static int xcs_ctrl_fd = -1; /* connection to the xcs server. */ +static int xcs_data_fd = -1; /* connection to the xcs server. */ + +int tcp_connect(char *ip, short port) +{ + struct sockaddr_in addr; + int ret, fd; + + fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) + { + printf("error creating xcs socket!\n"); + return -1; + } + + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = inet_addr(ip); + memset(&(addr.sin_zero), '\0', 8); + + ret = connect(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr)); + if (ret < 0) + { + printf("error connecting to xcs!\n"); + return -1; + } + + return fd; +} + +void tcp_disconnect(int *fd) +{ + close(*fd); + *fd = -1; +} + +void xcs_read(int fd, xcs_msg_t *msg) +{ + int ret; + + ret = read(fd, msg, sizeof(xcs_msg_t)); + if (ret != sizeof(xcs_msg_t)) { + printf("read error\n"); + exit(-1); + } +} + +void xcs_send(int fd, xcs_msg_t *msg) +{ + int ret; + + ret = send(fd, msg, sizeof(xcs_msg_t), 0); + if (ret != sizeof(xcs_msg_t) ) + { + printf("send error\n"); + exit(-1); + } +} + + +int main(int argc, char* argv[]) +{ + int ret, i; + xcs_msg_t msg; + control_msg_t *cmsg; + int verbose = 0; + + if (argc > 1) + if ((strlen(argv[1]) >=2) && (strncmp(argv[1], "-v", 2) == 0)) + verbose = 1; + + ret = tcp_connect("127.0.0.1", XCS_TCP_PORT); + if (ret < 0) + { + printf("connect failed!\n"); + exit(-1); + } + xcs_ctrl_fd = ret; + + memset(&msg, 0, sizeof(msg)); + msg.type = XCS_CONNECT_CTRL; + xcs_send(xcs_ctrl_fd, &msg); + xcs_read(xcs_ctrl_fd, &msg); + if (msg.result != XCS_RSLT_OK) + { + printf("Error connecting control channel\n"); + exit(-1); + } + + ret = tcp_connect("127.0.0.1", XCS_TCP_PORT); + if (ret < 0) + { + printf("connect failed!\n"); + exit(-1); + } + xcs_data_fd = ret; + + msg.type = XCS_CONNECT_DATA; + /* session id is set from before... */ + xcs_send(xcs_data_fd, &msg); + xcs_read(xcs_data_fd, &msg); + if (msg.result != XCS_RSLT_OK) + { + printf("Error connecting data channel\n"); + exit(-1); + } + + msg.type = XCS_MSG_BIND; + msg.u.bind.port = PORT_WILDCARD; + msg.u.bind.type = TYPE_WILDCARD; + xcs_send(xcs_ctrl_fd, &msg); + xcs_read(xcs_ctrl_fd, &msg); + if (msg.result != XCS_RSLT_OK) + { + printf("Error binding.\n"); + exit(-1); + } + + + while (1) + { + xcs_read(xcs_data_fd, &msg); + cmsg = &msg.u.control.msg; + + for (i=0; i<60; i++) + if ((!isprint(cmsg->msg[i])) && (cmsg->msg[i] != '\0')) + cmsg->msg[i] = '.'; + cmsg->msg[59] = '\0'; + + switch (msg.type) + { + case XCS_REQUEST: + printf("[REQUEST ] : (dom:%u port:%d) (type:(%d,%d) len %d) \n", + msg.u.control.remote_dom, + msg.u.control.local_port, + msg.u.control.msg.type, + msg.u.control.msg.subtype, + msg.u.control.msg.length); + if (verbose) + printf(" : %s\n", msg.u.control.msg.msg); + break; + case XCS_RESPONSE: + printf("[RESPONSE] : (dom:%u port:%d) (type:(%d,%d) len %d) \n", + msg.u.control.remote_dom, + msg.u.control.local_port, + msg.u.control.msg.type, + msg.u.control.msg.subtype, + msg.u.control.msg.length); + if (verbose) + printf(" : %s\n", msg.u.control.msg.msg); + break; + case XCS_VIRQ: + printf("[VIRQ ] : %d\n", msg.u.control.local_port); + default: + printf("[UNKNOWN ]\n"); + } + } + + return(0); +} diff --git a/xen/include/public/io/domain_controller.h b/xen/include/public/io/domain_controller.h index c248f21419..1f16d4f2f8 100644 --- a/xen/include/public/io/domain_controller.h +++ b/xen/include/public/io/domain_controller.h @@ -10,6 +10,7 @@ #ifndef __XEN_PUBLIC_IO_DOMAIN_CONTROLLER_H__ #define __XEN_PUBLIC_IO_DOMAIN_CONTROLLER_H__ +#include "ring.h" /* * Reason codes for SCHEDOP_shutdown. These are opaque to Xen but may be @@ -33,15 +34,24 @@ typedef struct { u8 msg[60]; /* 4: type-specific message data */ } PACKED control_msg_t; /* 64 bytes */ +/* These are used by the control message deferred ring. */ #define CONTROL_RING_SIZE 8 typedef u32 CONTROL_RING_IDX; #define MASK_CONTROL_IDX(_i) ((_i)&(CONTROL_RING_SIZE-1)) +/* + * Generate control ring structures and types. + * + * CONTROL_RING_MEM is currently an 8-slot ring of ctrl_msg_t structs and + * two 32-bit counters: (64 * 8) + (2 * 4) = 520 + */ +#define CONTROL_RING_MEM 520 +#define CTRL_RING RING_PARAMS(control_msg_t, control_msg_t, CONTROL_RING_MEM) +DEFINE_RING_TYPES(ctrl, CTRL_RING); + typedef struct { - control_msg_t tx_ring[CONTROL_RING_SIZE]; /* 0: guest -> controller */ - control_msg_t rx_ring[CONTROL_RING_SIZE]; /* 512: controller -> guest */ - CONTROL_RING_IDX tx_req_prod, tx_resp_prod; /* 1024, 1028 */ - CONTROL_RING_IDX rx_req_prod, rx_resp_prod; /* 1032, 1036 */ + ctrl_sring_t tx_ring; /* 0: guest -> controller */ + ctrl_sring_t rx_ring; /* 520: controller -> guest */ } PACKED control_if_t; /* 1040 bytes */ /* -- 2.30.2